diff --git a/experiments/think-d12-1ep-25sh-r11-wd42/run.json b/experiments/think-d12-1ep-25sh-r11-wd42/run.json new file mode 100644 index 0000000000000000000000000000000000000000..5263e8b5c526a64212bdbfd68eca0b704ea98092 --- /dev/null +++ b/experiments/think-d12-1ep-25sh-r11-wd42/run.json @@ -0,0 +1,10 @@ +{ + "experiment_id": "think-d12-1ep-25sh-r11-wd42", + "stage": "base", + "base_experiment_id": "think-d12-1ep-25sh-r11-wd42", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "config_fingerprint": "47bfa49108766b7d", + "wandb_run_id": "4e526b5a", + "created_at": 1782495190 +} diff --git a/experiments/think-d12-1ep-25sh-r11-wd42/summary.json b/experiments/think-d12-1ep-25sh-r11-wd42/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..2175bed11026365f7d85223d427e8fdc4f5fd599 --- /dev/null +++ b/experiments/think-d12-1ep-25sh-r11-wd42/summary.json @@ -0,0 +1,32 @@ +{ + "experiment_id": "think-d12-1ep-25sh-r11-wd42", + "stage": "base", + "base_experiment_id": "think-d12-1ep-25sh-r11-wd42", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "dataset": "jbduran/think-dataset", + "dataset_revision": "main", + "step": 2362, + "depth": 12, + "target_param_data_ratio": 11.25, + "training_tokens": 1238368256, + "final_sampled_val_bpb": 1.103670265641304, + "minimum_sampled_val_bpb": 1.103670265641304, + "full_val_bpb": 1.0526348691238439, + "core_metric": null, + "centered_results": null, + "conditioned_samples": [], + "unconditioned_samples": [], + "training_time_seconds": 6293.967695713043, + "stage_training_flops": 1.0985538644638433e+18, + "inherited_parent_flops": 0.0, + "cumulative_pipeline_training_flops": 1.0985538644638433e+18, + "config_fingerprint": "47bfa49108766b7d", + "git_commit_sha": "7e29503cca7b67e1418323c3628e6f935e4feacf", + "wandb_url": "https://wandb.ai/jbduran-thinkingmachinesncsu/think.nano/runs/4e526b5a", + "huggingface_url": "https://huggingface.co/jbduran/think.nano/tree/main/experiments/think-d12-1ep-25sh-r11-wd42", + "dataset_fingerprint": "a6e1b3a100e0d8b3", + "tokenizer_fingerprint": "ebb3705d7792a34d", + "unique_train_tokens": 1275519304, + "effective_epochs": 0.970873786164196 +} diff --git a/experiments/think-d12-1ep-25sh-r11-wd42/tokenizer/experiment_tokenizer.json b/experiments/think-d12-1ep-25sh-r11-wd42/tokenizer/experiment_tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..01564e84d532ee9e205955aaffee1d865c4b28f0 --- /dev/null +++ b/experiments/think-d12-1ep-25sh-r11-wd42/tokenizer/experiment_tokenizer.json @@ -0,0 +1,18 @@ +{ + "experiment_id": "think-d12-1ep-25sh-r11-wd42", + "dataset": { + "adapter": "parquet_shards", + "repo": "jbduran/think-dataset", + "revision": "main", + "validation_shard": 472, + "num_train_shards": 44, + "download_workers": 4 + }, + "tokenizer": { + "mode": "train", + "max_chars": 2000000000, + "doc_cap": 10000, + "vocab_size": 32768 + }, + "created_at": 1782495209 +} diff --git a/experiments/think-d12-1ep-25sh-r11-wd42/tokenizer/token_bytes.pt b/experiments/think-d12-1ep-25sh-r11-wd42/tokenizer/token_bytes.pt new file mode 100644 index 0000000000000000000000000000000000000000..80dbb386d071538021ab399ee6e965ae0cd1a54e --- /dev/null +++ b/experiments/think-d12-1ep-25sh-r11-wd42/tokenizer/token_bytes.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:59f928e04aa2ac37dd4064493240d1e73ecab7acb217c5a183311b0c523a3468 +size 132649 diff --git a/experiments/think-d12-1ep-25sh-r11-wd42/tokenizer/tokenizer.pkl b/experiments/think-d12-1ep-25sh-r11-wd42/tokenizer/tokenizer.pkl new file mode 100644 index 0000000000000000000000000000000000000000..a812730c3de1acc1e9e30ef6c4ccf1e9360f5d8a --- /dev/null +++ b/experiments/think-d12-1ep-25sh-r11-wd42/tokenizer/tokenizer.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fd583e3c35851c62295a1a0f6d688923e4f30649ac963b701ec2440fe8bc3e4f +size 404221 diff --git a/experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints/meta_000500.json b/experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints/meta_000500.json new file mode 100644 index 0000000000000000000000000000000000000000..59f658d8c9d9655b7ddf9466f4304ecc0d20e9d4 --- /dev/null +++ b/experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints/meta_000500.json @@ -0,0 +1,141 @@ +{ + "step": 500, + "experiment_id": "think-d12-1ep-44sh-r20-wd42", + "val_bpb": 1.3221519749129604, + "model_config": { + "sequence_len": 2048, + "vocab_size": 32768, + "n_layer": 12, + "n_head": 6, + "n_kv_head": 6, + "n_embd": 768, + "window_pattern": "L" + }, + "user_config": { + "run": "think-d12-1ep-44sh-r20-wd42", + "wandb_run_id": "5c4fba8a", + "wandb_group": "think-d12", + "wandb_tags": "think-dataset,d12,ratio20,44shards,1epoch,wd.42", + "device_type": "", + "fp8": false, + "fp8_recipe": "tensorwise", + "depth": 12, + "aspect_ratio": 64, + "head_dim": 128, + "max_seq_len": 2048, + "window_pattern": "L", + "num_iterations": -1, + "target_flops": -1.0, + "target_param_data_ratio": 20.0, + "device_batch_size": 16, + "total_batch_size": 524288, + "embedding_lr": 0.3, + "unembedding_lr": 0.008, + "weight_decay": 0.42, + "matrix_lr": 0.02, + "scalar_lr": 0.5, + "warmup_steps": 40, + "warmdown_ratio": 0.65, + "final_lr_frac": 0.05, + "resume_from_step": -1, + "pretokenized": true, + "data_dir": "/content/nanochat_cache/experiments/think-d12-1ep-44sh-r20-wd42/data", + "tokenizer_dir": "/content/nanochat_cache/experiments/think-d12-1ep-44sh-r20-wd42/tokenizer", + "pretokenized_dir": "/content/nanochat_cache/experiments/think-d12-1ep-44sh-r20-wd42/pretok", + "checkpoint_dir": "/content/nanochat_cache/experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints", + "experiment_id": "think-d12-1ep-44sh-r20-wd42", + "experiment_config": "/content/nanochat_cache/experiments/think-d12-1ep-44sh-r20-wd42/config.json", + "tokenizer_fingerprint": "1744d7b7ee0d5d80", + "git_commit_sha": "fd4d149a650d0fb07ae9b96864bb40e5769622d0", + "seed": 42, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "core_metric_max_per_task": 500, + "sample_every": -1, + "save_every": 500, + "model_tag": "think-d12-1ep-44sh-r20-wd42", + "experiment": { + "schema_version": 1, + "stage": "base", + "experiment_id": "think-d12-1ep-44sh-r20-wd42", + "dataset": { + "adapter": "parquet_shards", + "repo": "jbduran/think-dataset", + "revision": "main", + "validation_shard": 472, + "num_train_shards": 44, + "download_workers": 4 + }, + "tokenizer": { + "mode": "train", + "max_chars": 2000000000, + "doc_cap": 10000, + "vocab_size": 32768 + }, + "pretokenize": { + "enabled": true, + "slack": 1.03, + "val_tokens": 20971520, + "shard_tokens": 100000000, + "tokenizer_threads": 8 + }, + "training": { + "depth": 12, + "scaling_params": 110100912, + "target_param_data_ratio": 20.0, + "window_pattern": "L", + "weight_decay": 0.42, + "device_batch_size": 16, + "total_batch_size": 524288, + "save_every": 500, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "sample_every": -1 + }, + "artifacts": { + "repo": "jbduran/think.nano" + }, + "wandb": { + "entity": "jbduran-thinkingmachinesncsu", + "project": "think.nano", + "name": "think-d12-1ep-44sh-r20-wd42", + "group": "think-d12", + "tags": [ + "think-dataset", + "d12", + "ratio20", + "44shards", + "1epoch", + "wd.42" + ] + }, + "config_fingerprint": "a8d632854c2cd1bd", + "artifact_path": "experiments/think-d12-1ep-44sh-r20-wd42" + }, + "stage": "base", + "base_experiment_id": "think-d12-1ep-44sh-r20-wd42", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "config_fingerprint": "a8d632854c2cd1bd" + }, + "device_batch_size": 16, + "max_seq_len": 2048, + "total_batch_size": 524288, + "dataloader_state_dict": { + "file_idx": 2, + "pos": 62184769, + "epoch": 1, + "pq_idx": 2, + "rg_idx": 62184769 + }, + "loop_state": { + "min_val_bpb": 1.3221519749129604, + "smooth_train_loss": 3.7478778179789605, + "total_training_time": 1305.7518684864044, + "stage_training_flops": 232547388751872000, + "inherited_parent_flops": 0.0, + "cumulative_pipeline_training_flops": 232547388751872000 + } +} \ No newline at end of file diff --git a/experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints/meta_001000.json b/experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints/meta_001000.json new file mode 100644 index 0000000000000000000000000000000000000000..7ad198c79884fcd6bfa13ebc8fb085602b7b4960 --- /dev/null +++ b/experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints/meta_001000.json @@ -0,0 +1,141 @@ +{ + "step": 1000, + "experiment_id": "think-d12-1ep-44sh-r20-wd42", + "val_bpb": 1.2600323113203347, + "model_config": { + "sequence_len": 2048, + "vocab_size": 32768, + "n_layer": 12, + "n_head": 6, + "n_kv_head": 6, + "n_embd": 768, + "window_pattern": "L" + }, + "user_config": { + "run": "think-d12-1ep-44sh-r20-wd42", + "wandb_run_id": "5c4fba8a", + "wandb_group": "think-d12", + "wandb_tags": "think-dataset,d12,ratio20,44shards,1epoch,wd.42", + "device_type": "", + "fp8": false, + "fp8_recipe": "tensorwise", + "depth": 12, + "aspect_ratio": 64, + "head_dim": 128, + "max_seq_len": 2048, + "window_pattern": "L", + "num_iterations": -1, + "target_flops": -1.0, + "target_param_data_ratio": 20.0, + "device_batch_size": 16, + "total_batch_size": 524288, + "embedding_lr": 0.3, + "unembedding_lr": 0.008, + "weight_decay": 0.42, + "matrix_lr": 0.02, + "scalar_lr": 0.5, + "warmup_steps": 40, + "warmdown_ratio": 0.65, + "final_lr_frac": 0.05, + "resume_from_step": -1, + "pretokenized": true, + "data_dir": "/content/nanochat_cache/experiments/think-d12-1ep-44sh-r20-wd42/data", + "tokenizer_dir": "/content/nanochat_cache/experiments/think-d12-1ep-44sh-r20-wd42/tokenizer", + "pretokenized_dir": "/content/nanochat_cache/experiments/think-d12-1ep-44sh-r20-wd42/pretok", + "checkpoint_dir": "/content/nanochat_cache/experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints", + "experiment_id": "think-d12-1ep-44sh-r20-wd42", + "experiment_config": "/content/nanochat_cache/experiments/think-d12-1ep-44sh-r20-wd42/config.json", + "tokenizer_fingerprint": "1744d7b7ee0d5d80", + "git_commit_sha": "fd4d149a650d0fb07ae9b96864bb40e5769622d0", + "seed": 42, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "core_metric_max_per_task": 500, + "sample_every": -1, + "save_every": 500, + "model_tag": "think-d12-1ep-44sh-r20-wd42", + "experiment": { + "schema_version": 1, + "stage": "base", + "experiment_id": "think-d12-1ep-44sh-r20-wd42", + "dataset": { + "adapter": "parquet_shards", + "repo": "jbduran/think-dataset", + "revision": "main", + "validation_shard": 472, + "num_train_shards": 44, + "download_workers": 4 + }, + "tokenizer": { + "mode": "train", + "max_chars": 2000000000, + "doc_cap": 10000, + "vocab_size": 32768 + }, + "pretokenize": { + "enabled": true, + "slack": 1.03, + "val_tokens": 20971520, + "shard_tokens": 100000000, + "tokenizer_threads": 8 + }, + "training": { + "depth": 12, + "scaling_params": 110100912, + "target_param_data_ratio": 20.0, + "window_pattern": "L", + "weight_decay": 0.42, + "device_batch_size": 16, + "total_batch_size": 524288, + "save_every": 500, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "sample_every": -1 + }, + "artifacts": { + "repo": "jbduran/think.nano" + }, + "wandb": { + "entity": "jbduran-thinkingmachinesncsu", + "project": "think.nano", + "name": "think-d12-1ep-44sh-r20-wd42", + "group": "think-d12", + "tags": [ + "think-dataset", + "d12", + "ratio20", + "44shards", + "1epoch", + "wd.42" + ] + }, + "config_fingerprint": "a8d632854c2cd1bd", + "artifact_path": "experiments/think-d12-1ep-44sh-r20-wd42" + }, + "stage": "base", + "base_experiment_id": "think-d12-1ep-44sh-r20-wd42", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "config_fingerprint": "a8d632854c2cd1bd" + }, + "device_batch_size": 16, + "max_seq_len": 2048, + "total_batch_size": 524288, + "dataloader_state_dict": { + "file_idx": 5, + "pos": 24336769, + "epoch": 1, + "pq_idx": 5, + "rg_idx": 24336769 + }, + "loop_state": { + "min_val_bpb": 1.2600323113203347, + "smooth_train_loss": 3.646794584039554, + "total_training_time": 2640.877459049225, + "stage_training_flops": 465094777503744000, + "inherited_parent_flops": 0.0, + "cumulative_pipeline_training_flops": 465094777503744000 + } +} \ No newline at end of file diff --git a/experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints/meta_001500.json b/experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints/meta_001500.json new file mode 100644 index 0000000000000000000000000000000000000000..9f43a27678e300b6474f88abba476e3cae89d80a --- /dev/null +++ b/experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints/meta_001500.json @@ -0,0 +1,141 @@ +{ + "step": 1500, + "experiment_id": "think-d12-1ep-44sh-r20-wd42", + "val_bpb": 1.2377503150891866, + "model_config": { + "sequence_len": 2048, + "vocab_size": 32768, + "n_layer": 12, + "n_head": 6, + "n_kv_head": 6, + "n_embd": 768, + "window_pattern": "L" + }, + "user_config": { + "run": "think-d12-1ep-44sh-r20-wd42", + "wandb_run_id": "5c4fba8a", + "wandb_group": "think-d12", + "wandb_tags": "think-dataset,d12,ratio20,44shards,1epoch,wd.42", + "device_type": "", + "fp8": false, + "fp8_recipe": "tensorwise", + "depth": 12, + "aspect_ratio": 64, + "head_dim": 128, + "max_seq_len": 2048, + "window_pattern": "L", + "num_iterations": -1, + "target_flops": -1.0, + "target_param_data_ratio": 20.0, + "device_batch_size": 16, + "total_batch_size": 524288, + "embedding_lr": 0.3, + "unembedding_lr": 0.008, + "weight_decay": 0.42, + "matrix_lr": 0.02, + "scalar_lr": 0.5, + "warmup_steps": 40, + "warmdown_ratio": 0.65, + "final_lr_frac": 0.05, + "resume_from_step": -1, + "pretokenized": true, + "data_dir": "/content/nanochat_cache/experiments/think-d12-1ep-44sh-r20-wd42/data", + "tokenizer_dir": "/content/nanochat_cache/experiments/think-d12-1ep-44sh-r20-wd42/tokenizer", + "pretokenized_dir": "/content/nanochat_cache/experiments/think-d12-1ep-44sh-r20-wd42/pretok", + "checkpoint_dir": "/content/nanochat_cache/experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints", + "experiment_id": "think-d12-1ep-44sh-r20-wd42", + "experiment_config": "/content/nanochat_cache/experiments/think-d12-1ep-44sh-r20-wd42/config.json", + "tokenizer_fingerprint": "1744d7b7ee0d5d80", + "git_commit_sha": "fd4d149a650d0fb07ae9b96864bb40e5769622d0", + "seed": 42, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "core_metric_max_per_task": 500, + "sample_every": -1, + "save_every": 500, + "model_tag": "think-d12-1ep-44sh-r20-wd42", + "experiment": { + "schema_version": 1, + "stage": "base", + "experiment_id": "think-d12-1ep-44sh-r20-wd42", + "dataset": { + "adapter": "parquet_shards", + "repo": "jbduran/think-dataset", + "revision": "main", + "validation_shard": 472, + "num_train_shards": 44, + "download_workers": 4 + }, + "tokenizer": { + "mode": "train", + "max_chars": 2000000000, + "doc_cap": 10000, + "vocab_size": 32768 + }, + "pretokenize": { + "enabled": true, + "slack": 1.03, + "val_tokens": 20971520, + "shard_tokens": 100000000, + "tokenizer_threads": 8 + }, + "training": { + "depth": 12, + "scaling_params": 110100912, + "target_param_data_ratio": 20.0, + "window_pattern": "L", + "weight_decay": 0.42, + "device_batch_size": 16, + "total_batch_size": 524288, + "save_every": 500, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "sample_every": -1 + }, + "artifacts": { + "repo": "jbduran/think.nano" + }, + "wandb": { + "entity": "jbduran-thinkingmachinesncsu", + "project": "think.nano", + "name": "think-d12-1ep-44sh-r20-wd42", + "group": "think-d12", + "tags": [ + "think-dataset", + "d12", + "ratio20", + "44shards", + "1epoch", + "wd.42" + ] + }, + "config_fingerprint": "a8d632854c2cd1bd", + "artifact_path": "experiments/think-d12-1ep-44sh-r20-wd42" + }, + "stage": "base", + "base_experiment_id": "think-d12-1ep-44sh-r20-wd42", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "config_fingerprint": "a8d632854c2cd1bd" + }, + "device_batch_size": 16, + "max_seq_len": 2048, + "total_batch_size": 524288, + "dataloader_state_dict": { + "file_idx": 7, + "pos": 86488769, + "epoch": 1, + "pq_idx": 7, + "rg_idx": 86488769 + }, + "loop_state": { + "min_val_bpb": 1.2377503150891866, + "smooth_train_loss": 3.6301349812757526, + "total_training_time": 3977.6890711784363, + "stage_training_flops": 697642166255616000, + "inherited_parent_flops": 0.0, + "cumulative_pipeline_training_flops": 697642166255616000 + } +} \ No newline at end of file diff --git a/experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints/meta_002000.json b/experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints/meta_002000.json new file mode 100644 index 0000000000000000000000000000000000000000..16044dcb616400cb3c75f4feeed101b1751a8069 --- /dev/null +++ b/experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints/meta_002000.json @@ -0,0 +1,141 @@ +{ + "step": 2000, + "experiment_id": "think-d12-1ep-44sh-r20-wd42", + "val_bpb": 1.2014593386689274, + "model_config": { + "sequence_len": 2048, + "vocab_size": 32768, + "n_layer": 12, + "n_head": 6, + "n_kv_head": 6, + "n_embd": 768, + "window_pattern": "L" + }, + "user_config": { + "run": "think-d12-1ep-44sh-r20-wd42", + "wandb_run_id": "5c4fba8a", + "wandb_group": "think-d12", + "wandb_tags": "think-dataset,d12,ratio20,44shards,1epoch,wd.42", + "device_type": "", + "fp8": false, + "fp8_recipe": "tensorwise", + "depth": 12, + "aspect_ratio": 64, + "head_dim": 128, + "max_seq_len": 2048, + "window_pattern": "L", + "num_iterations": -1, + "target_flops": -1.0, + "target_param_data_ratio": 20.0, + "device_batch_size": 16, + "total_batch_size": 524288, + "embedding_lr": 0.3, + "unembedding_lr": 0.008, + "weight_decay": 0.42, + "matrix_lr": 0.02, + "scalar_lr": 0.5, + "warmup_steps": 40, + "warmdown_ratio": 0.65, + "final_lr_frac": 0.05, + "resume_from_step": -1, + "pretokenized": true, + "data_dir": "/content/nanochat_cache/experiments/think-d12-1ep-44sh-r20-wd42/data", + "tokenizer_dir": "/content/nanochat_cache/experiments/think-d12-1ep-44sh-r20-wd42/tokenizer", + "pretokenized_dir": "/content/nanochat_cache/experiments/think-d12-1ep-44sh-r20-wd42/pretok", + "checkpoint_dir": "/content/nanochat_cache/experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints", + "experiment_id": "think-d12-1ep-44sh-r20-wd42", + "experiment_config": "/content/nanochat_cache/experiments/think-d12-1ep-44sh-r20-wd42/config.json", + "tokenizer_fingerprint": "1744d7b7ee0d5d80", + "git_commit_sha": "fd4d149a650d0fb07ae9b96864bb40e5769622d0", + "seed": 42, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "core_metric_max_per_task": 500, + "sample_every": -1, + "save_every": 500, + "model_tag": "think-d12-1ep-44sh-r20-wd42", + "experiment": { + "schema_version": 1, + "stage": "base", + "experiment_id": "think-d12-1ep-44sh-r20-wd42", + "dataset": { + "adapter": "parquet_shards", + "repo": "jbduran/think-dataset", + "revision": "main", + "validation_shard": 472, + "num_train_shards": 44, + "download_workers": 4 + }, + "tokenizer": { + "mode": "train", + "max_chars": 2000000000, + "doc_cap": 10000, + "vocab_size": 32768 + }, + "pretokenize": { + "enabled": true, + "slack": 1.03, + "val_tokens": 20971520, + "shard_tokens": 100000000, + "tokenizer_threads": 8 + }, + "training": { + "depth": 12, + "scaling_params": 110100912, + "target_param_data_ratio": 20.0, + "window_pattern": "L", + "weight_decay": 0.42, + "device_batch_size": 16, + "total_batch_size": 524288, + "save_every": 500, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "sample_every": -1 + }, + "artifacts": { + "repo": "jbduran/think.nano" + }, + "wandb": { + "entity": "jbduran-thinkingmachinesncsu", + "project": "think.nano", + "name": "think-d12-1ep-44sh-r20-wd42", + "group": "think-d12", + "tags": [ + "think-dataset", + "d12", + "ratio20", + "44shards", + "1epoch", + "wd.42" + ] + }, + "config_fingerprint": "a8d632854c2cd1bd", + "artifact_path": "experiments/think-d12-1ep-44sh-r20-wd42" + }, + "stage": "base", + "base_experiment_id": "think-d12-1ep-44sh-r20-wd42", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "config_fingerprint": "a8d632854c2cd1bd" + }, + "device_batch_size": 16, + "max_seq_len": 2048, + "total_batch_size": 524288, + "dataloader_state_dict": { + "file_idx": 10, + "pos": 48640769, + "epoch": 1, + "pq_idx": 10, + "rg_idx": 48640769 + }, + "loop_state": { + "min_val_bpb": 1.2014593386689274, + "smooth_train_loss": 3.4382217869051193, + "total_training_time": 5322.514421463013, + "stage_training_flops": 930189555007488000, + "inherited_parent_flops": 0.0, + "cumulative_pipeline_training_flops": 930189555007488000 + } +} \ No newline at end of file diff --git a/experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints/meta_002500.json b/experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints/meta_002500.json new file mode 100644 index 0000000000000000000000000000000000000000..4c7afc7891e2611d2ce054d188a3d16ed39f14ae --- /dev/null +++ b/experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints/meta_002500.json @@ -0,0 +1,141 @@ +{ + "step": 2500, + "experiment_id": "think-d12-1ep-44sh-r20-wd42", + "val_bpb": 1.166274403385848, + "model_config": { + "sequence_len": 2048, + "vocab_size": 32768, + "n_layer": 12, + "n_head": 6, + "n_kv_head": 6, + "n_embd": 768, + "window_pattern": "L" + }, + "user_config": { + "run": "think-d12-1ep-44sh-r20-wd42", + "wandb_run_id": "5c4fba8a", + "wandb_group": "think-d12", + "wandb_tags": "think-dataset,d12,ratio20,44shards,1epoch,wd.42", + "device_type": "", + "fp8": false, + "fp8_recipe": "tensorwise", + "depth": 12, + "aspect_ratio": 64, + "head_dim": 128, + "max_seq_len": 2048, + "window_pattern": "L", + "num_iterations": -1, + "target_flops": -1.0, + "target_param_data_ratio": 20.0, + "device_batch_size": 16, + "total_batch_size": 524288, + "embedding_lr": 0.3, + "unembedding_lr": 0.008, + "weight_decay": 0.42, + "matrix_lr": 0.02, + "scalar_lr": 0.5, + "warmup_steps": 40, + "warmdown_ratio": 0.65, + "final_lr_frac": 0.05, + "resume_from_step": -1, + "pretokenized": true, + "data_dir": "/content/nanochat_cache/experiments/think-d12-1ep-44sh-r20-wd42/data", + "tokenizer_dir": "/content/nanochat_cache/experiments/think-d12-1ep-44sh-r20-wd42/tokenizer", + "pretokenized_dir": "/content/nanochat_cache/experiments/think-d12-1ep-44sh-r20-wd42/pretok", + "checkpoint_dir": "/content/nanochat_cache/experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints", + "experiment_id": "think-d12-1ep-44sh-r20-wd42", + "experiment_config": "/content/nanochat_cache/experiments/think-d12-1ep-44sh-r20-wd42/config.json", + "tokenizer_fingerprint": "1744d7b7ee0d5d80", + "git_commit_sha": "fd4d149a650d0fb07ae9b96864bb40e5769622d0", + "seed": 42, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "core_metric_max_per_task": 500, + "sample_every": -1, + "save_every": 500, + "model_tag": "think-d12-1ep-44sh-r20-wd42", + "experiment": { + "schema_version": 1, + "stage": "base", + "experiment_id": "think-d12-1ep-44sh-r20-wd42", + "dataset": { + "adapter": "parquet_shards", + "repo": "jbduran/think-dataset", + "revision": "main", + "validation_shard": 472, + "num_train_shards": 44, + "download_workers": 4 + }, + "tokenizer": { + "mode": "train", + "max_chars": 2000000000, + "doc_cap": 10000, + "vocab_size": 32768 + }, + "pretokenize": { + "enabled": true, + "slack": 1.03, + "val_tokens": 20971520, + "shard_tokens": 100000000, + "tokenizer_threads": 8 + }, + "training": { + "depth": 12, + "scaling_params": 110100912, + "target_param_data_ratio": 20.0, + "window_pattern": "L", + "weight_decay": 0.42, + "device_batch_size": 16, + "total_batch_size": 524288, + "save_every": 500, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "sample_every": -1 + }, + "artifacts": { + "repo": "jbduran/think.nano" + }, + "wandb": { + "entity": "jbduran-thinkingmachinesncsu", + "project": "think.nano", + "name": "think-d12-1ep-44sh-r20-wd42", + "group": "think-d12", + "tags": [ + "think-dataset", + "d12", + "ratio20", + "44shards", + "1epoch", + "wd.42" + ] + }, + "config_fingerprint": "a8d632854c2cd1bd", + "artifact_path": "experiments/think-d12-1ep-44sh-r20-wd42" + }, + "stage": "base", + "base_experiment_id": "think-d12-1ep-44sh-r20-wd42", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "config_fingerprint": "a8d632854c2cd1bd" + }, + "device_batch_size": 16, + "max_seq_len": 2048, + "total_batch_size": 524288, + "dataloader_state_dict": { + "file_idx": 13, + "pos": 10792769, + "epoch": 1, + "pq_idx": 13, + "rg_idx": 10792769 + }, + "loop_state": { + "min_val_bpb": 1.166274403385848, + "smooth_train_loss": 3.265478801787732, + "total_training_time": 6658.766751766205, + "stage_training_flops": 1162736943759360000, + "inherited_parent_flops": 0.0, + "cumulative_pipeline_training_flops": 1162736943759360000 + } +} \ No newline at end of file diff --git a/experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints/meta_003000.json b/experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints/meta_003000.json new file mode 100644 index 0000000000000000000000000000000000000000..c8041b4c09d2307652cf533a20e28d08de809019 --- /dev/null +++ b/experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints/meta_003000.json @@ -0,0 +1,141 @@ +{ + "step": 3000, + "experiment_id": "think-d12-1ep-44sh-r20-wd42", + "val_bpb": 1.1420328141160099, + "model_config": { + "sequence_len": 2048, + "vocab_size": 32768, + "n_layer": 12, + "n_head": 6, + "n_kv_head": 6, + "n_embd": 768, + "window_pattern": "L" + }, + "user_config": { + "run": "think-d12-1ep-44sh-r20-wd42", + "wandb_run_id": "5c4fba8a", + "wandb_group": "think-d12", + "wandb_tags": "think-dataset,d12,ratio20,44shards,1epoch,wd.42", + "device_type": "", + "fp8": false, + "fp8_recipe": "tensorwise", + "depth": 12, + "aspect_ratio": 64, + "head_dim": 128, + "max_seq_len": 2048, + "window_pattern": "L", + "num_iterations": -1, + "target_flops": -1.0, + "target_param_data_ratio": 20.0, + "device_batch_size": 16, + "total_batch_size": 524288, + "embedding_lr": 0.3, + "unembedding_lr": 0.008, + "weight_decay": 0.42, + "matrix_lr": 0.02, + "scalar_lr": 0.5, + "warmup_steps": 40, + "warmdown_ratio": 0.65, + "final_lr_frac": 0.05, + "resume_from_step": -1, + "pretokenized": true, + "data_dir": "/content/nanochat_cache/experiments/think-d12-1ep-44sh-r20-wd42/data", + "tokenizer_dir": "/content/nanochat_cache/experiments/think-d12-1ep-44sh-r20-wd42/tokenizer", + "pretokenized_dir": "/content/nanochat_cache/experiments/think-d12-1ep-44sh-r20-wd42/pretok", + "checkpoint_dir": "/content/nanochat_cache/experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints", + "experiment_id": "think-d12-1ep-44sh-r20-wd42", + "experiment_config": "/content/nanochat_cache/experiments/think-d12-1ep-44sh-r20-wd42/config.json", + "tokenizer_fingerprint": "1744d7b7ee0d5d80", + "git_commit_sha": "fd4d149a650d0fb07ae9b96864bb40e5769622d0", + "seed": 42, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "core_metric_max_per_task": 500, + "sample_every": -1, + "save_every": 500, + "model_tag": "think-d12-1ep-44sh-r20-wd42", + "experiment": { + "schema_version": 1, + "stage": "base", + "experiment_id": "think-d12-1ep-44sh-r20-wd42", + "dataset": { + "adapter": "parquet_shards", + "repo": "jbduran/think-dataset", + "revision": "main", + "validation_shard": 472, + "num_train_shards": 44, + "download_workers": 4 + }, + "tokenizer": { + "mode": "train", + "max_chars": 2000000000, + "doc_cap": 10000, + "vocab_size": 32768 + }, + "pretokenize": { + "enabled": true, + "slack": 1.03, + "val_tokens": 20971520, + "shard_tokens": 100000000, + "tokenizer_threads": 8 + }, + "training": { + "depth": 12, + "scaling_params": 110100912, + "target_param_data_ratio": 20.0, + "window_pattern": "L", + "weight_decay": 0.42, + "device_batch_size": 16, + "total_batch_size": 524288, + "save_every": 500, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "sample_every": -1 + }, + "artifacts": { + "repo": "jbduran/think.nano" + }, + "wandb": { + "entity": "jbduran-thinkingmachinesncsu", + "project": "think.nano", + "name": "think-d12-1ep-44sh-r20-wd42", + "group": "think-d12", + "tags": [ + "think-dataset", + "d12", + "ratio20", + "44shards", + "1epoch", + "wd.42" + ] + }, + "config_fingerprint": "a8d632854c2cd1bd", + "artifact_path": "experiments/think-d12-1ep-44sh-r20-wd42" + }, + "stage": "base", + "base_experiment_id": "think-d12-1ep-44sh-r20-wd42", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "config_fingerprint": "a8d632854c2cd1bd" + }, + "device_batch_size": 16, + "max_seq_len": 2048, + "total_batch_size": 524288, + "dataloader_state_dict": { + "file_idx": 15, + "pos": 72944769, + "epoch": 1, + "pq_idx": 15, + "rg_idx": 72944769 + }, + "loop_state": { + "min_val_bpb": 1.1420328141160099, + "smooth_train_loss": 3.1094901625575497, + "total_training_time": 8003.383926391602, + "stage_training_flops": 1395284332511232000, + "inherited_parent_flops": 0.0, + "cumulative_pipeline_training_flops": 1395284332511232000 + } +} \ No newline at end of file diff --git a/experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints/meta_003500.json b/experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints/meta_003500.json new file mode 100644 index 0000000000000000000000000000000000000000..c54a62105182496220f719255a66ed5fcd4ac91d --- /dev/null +++ b/experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints/meta_003500.json @@ -0,0 +1,141 @@ +{ + "step": 3500, + "experiment_id": "think-d12-1ep-44sh-r20-wd42", + "val_bpb": 1.1108293726096388, + "model_config": { + "sequence_len": 2048, + "vocab_size": 32768, + "n_layer": 12, + "n_head": 6, + "n_kv_head": 6, + "n_embd": 768, + "window_pattern": "L" + }, + "user_config": { + "run": "think-d12-1ep-44sh-r20-wd42", + "wandb_run_id": "5c4fba8a", + "wandb_group": "think-d12", + "wandb_tags": "think-dataset,d12,ratio20,44shards,1epoch,wd.42", + "device_type": "", + "fp8": false, + "fp8_recipe": "tensorwise", + "depth": 12, + "aspect_ratio": 64, + "head_dim": 128, + "max_seq_len": 2048, + "window_pattern": "L", + "num_iterations": -1, + "target_flops": -1.0, + "target_param_data_ratio": 20.0, + "device_batch_size": 16, + "total_batch_size": 524288, + "embedding_lr": 0.3, + "unembedding_lr": 0.008, + "weight_decay": 0.42, + "matrix_lr": 0.02, + "scalar_lr": 0.5, + "warmup_steps": 40, + "warmdown_ratio": 0.65, + "final_lr_frac": 0.05, + "resume_from_step": -1, + "pretokenized": true, + "data_dir": "/content/nanochat_cache/experiments/think-d12-1ep-44sh-r20-wd42/data", + "tokenizer_dir": "/content/nanochat_cache/experiments/think-d12-1ep-44sh-r20-wd42/tokenizer", + "pretokenized_dir": "/content/nanochat_cache/experiments/think-d12-1ep-44sh-r20-wd42/pretok", + "checkpoint_dir": "/content/nanochat_cache/experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints", + "experiment_id": "think-d12-1ep-44sh-r20-wd42", + "experiment_config": "/content/nanochat_cache/experiments/think-d12-1ep-44sh-r20-wd42/config.json", + "tokenizer_fingerprint": "1744d7b7ee0d5d80", + "git_commit_sha": "fd4d149a650d0fb07ae9b96864bb40e5769622d0", + "seed": 42, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "core_metric_max_per_task": 500, + "sample_every": -1, + "save_every": 500, + "model_tag": "think-d12-1ep-44sh-r20-wd42", + "experiment": { + "schema_version": 1, + "stage": "base", + "experiment_id": "think-d12-1ep-44sh-r20-wd42", + "dataset": { + "adapter": "parquet_shards", + "repo": "jbduran/think-dataset", + "revision": "main", + "validation_shard": 472, + "num_train_shards": 44, + "download_workers": 4 + }, + "tokenizer": { + "mode": "train", + "max_chars": 2000000000, + "doc_cap": 10000, + "vocab_size": 32768 + }, + "pretokenize": { + "enabled": true, + "slack": 1.03, + "val_tokens": 20971520, + "shard_tokens": 100000000, + "tokenizer_threads": 8 + }, + "training": { + "depth": 12, + "scaling_params": 110100912, + "target_param_data_ratio": 20.0, + "window_pattern": "L", + "weight_decay": 0.42, + "device_batch_size": 16, + "total_batch_size": 524288, + "save_every": 500, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "sample_every": -1 + }, + "artifacts": { + "repo": "jbduran/think.nano" + }, + "wandb": { + "entity": "jbduran-thinkingmachinesncsu", + "project": "think.nano", + "name": "think-d12-1ep-44sh-r20-wd42", + "group": "think-d12", + "tags": [ + "think-dataset", + "d12", + "ratio20", + "44shards", + "1epoch", + "wd.42" + ] + }, + "config_fingerprint": "a8d632854c2cd1bd", + "artifact_path": "experiments/think-d12-1ep-44sh-r20-wd42" + }, + "stage": "base", + "base_experiment_id": "think-d12-1ep-44sh-r20-wd42", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "config_fingerprint": "a8d632854c2cd1bd" + }, + "device_batch_size": 16, + "max_seq_len": 2048, + "total_batch_size": 524288, + "dataloader_state_dict": { + "file_idx": 18, + "pos": 35096769, + "epoch": 1, + "pq_idx": 18, + "rg_idx": 35096769 + }, + "loop_state": { + "min_val_bpb": 1.1108293726096388, + "smooth_train_loss": 3.0718457586789576, + "total_training_time": 9347.049030542374, + "stage_training_flops": 1627831721263104000, + "inherited_parent_flops": 0.0, + "cumulative_pipeline_training_flops": 1627831721263104000 + } +} \ No newline at end of file diff --git a/experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints/meta_004000.json b/experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints/meta_004000.json new file mode 100644 index 0000000000000000000000000000000000000000..8e7721ac608c77baaa5ec9bbe2c224e7ab18f6f8 --- /dev/null +++ b/experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints/meta_004000.json @@ -0,0 +1,141 @@ +{ + "step": 4000, + "experiment_id": "think-d12-1ep-44sh-r20-wd42", + "val_bpb": 1.0845218539469001, + "model_config": { + "sequence_len": 2048, + "vocab_size": 32768, + "n_layer": 12, + "n_head": 6, + "n_kv_head": 6, + "n_embd": 768, + "window_pattern": "L" + }, + "user_config": { + "run": "think-d12-1ep-44sh-r20-wd42", + "wandb_run_id": "5c4fba8a", + "wandb_group": "think-d12", + "wandb_tags": "think-dataset,d12,ratio20,44shards,1epoch,wd.42", + "device_type": "", + "fp8": false, + "fp8_recipe": "tensorwise", + "depth": 12, + "aspect_ratio": 64, + "head_dim": 128, + "max_seq_len": 2048, + "window_pattern": "L", + "num_iterations": -1, + "target_flops": -1.0, + "target_param_data_ratio": 20.0, + "device_batch_size": 16, + "total_batch_size": 524288, + "embedding_lr": 0.3, + "unembedding_lr": 0.008, + "weight_decay": 0.42, + "matrix_lr": 0.02, + "scalar_lr": 0.5, + "warmup_steps": 40, + "warmdown_ratio": 0.65, + "final_lr_frac": 0.05, + "resume_from_step": -1, + "pretokenized": true, + "data_dir": "/content/nanochat_cache/experiments/think-d12-1ep-44sh-r20-wd42/data", + "tokenizer_dir": "/content/nanochat_cache/experiments/think-d12-1ep-44sh-r20-wd42/tokenizer", + "pretokenized_dir": "/content/nanochat_cache/experiments/think-d12-1ep-44sh-r20-wd42/pretok", + "checkpoint_dir": "/content/nanochat_cache/experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints", + "experiment_id": "think-d12-1ep-44sh-r20-wd42", + "experiment_config": "/content/nanochat_cache/experiments/think-d12-1ep-44sh-r20-wd42/config.json", + "tokenizer_fingerprint": "1744d7b7ee0d5d80", + "git_commit_sha": "fd4d149a650d0fb07ae9b96864bb40e5769622d0", + "seed": 42, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "core_metric_max_per_task": 500, + "sample_every": -1, + "save_every": 500, + "model_tag": "think-d12-1ep-44sh-r20-wd42", + "experiment": { + "schema_version": 1, + "stage": "base", + "experiment_id": "think-d12-1ep-44sh-r20-wd42", + "dataset": { + "adapter": "parquet_shards", + "repo": "jbduran/think-dataset", + "revision": "main", + "validation_shard": 472, + "num_train_shards": 44, + "download_workers": 4 + }, + "tokenizer": { + "mode": "train", + "max_chars": 2000000000, + "doc_cap": 10000, + "vocab_size": 32768 + }, + "pretokenize": { + "enabled": true, + "slack": 1.03, + "val_tokens": 20971520, + "shard_tokens": 100000000, + "tokenizer_threads": 8 + }, + "training": { + "depth": 12, + "scaling_params": 110100912, + "target_param_data_ratio": 20.0, + "window_pattern": "L", + "weight_decay": 0.42, + "device_batch_size": 16, + "total_batch_size": 524288, + "save_every": 500, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "sample_every": -1 + }, + "artifacts": { + "repo": "jbduran/think.nano" + }, + "wandb": { + "entity": "jbduran-thinkingmachinesncsu", + "project": "think.nano", + "name": "think-d12-1ep-44sh-r20-wd42", + "group": "think-d12", + "tags": [ + "think-dataset", + "d12", + "ratio20", + "44shards", + "1epoch", + "wd.42" + ] + }, + "config_fingerprint": "a8d632854c2cd1bd", + "artifact_path": "experiments/think-d12-1ep-44sh-r20-wd42" + }, + "stage": "base", + "base_experiment_id": "think-d12-1ep-44sh-r20-wd42", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "config_fingerprint": "a8d632854c2cd1bd" + }, + "device_batch_size": 16, + "max_seq_len": 2048, + "total_batch_size": 524288, + "dataloader_state_dict": { + "file_idx": 20, + "pos": 97248769, + "epoch": 1, + "pq_idx": 20, + "rg_idx": 97248769 + }, + "loop_state": { + "min_val_bpb": 1.0845218539469001, + "smooth_train_loss": 2.94610128781119, + "total_training_time": 10685.518072605133, + "stage_training_flops": 1860379110014976000, + "inherited_parent_flops": 0.0, + "cumulative_pipeline_training_flops": 1860379110014976000 + } +} \ No newline at end of file diff --git a/experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints/meta_004200.json b/experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints/meta_004200.json new file mode 100644 index 0000000000000000000000000000000000000000..30c2ced2fd8737ce78266ad95b8535f0fc11d037 --- /dev/null +++ b/experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints/meta_004200.json @@ -0,0 +1,141 @@ +{ + "step": 4200, + "experiment_id": "think-d12-1ep-44sh-r20-wd42", + "val_bpb": 1.078128321044417, + "model_config": { + "sequence_len": 2048, + "vocab_size": 32768, + "n_layer": 12, + "n_head": 6, + "n_kv_head": 6, + "n_embd": 768, + "window_pattern": "L" + }, + "user_config": { + "run": "think-d12-1ep-44sh-r20-wd42", + "wandb_run_id": "5c4fba8a", + "wandb_group": "think-d12", + "wandb_tags": "think-dataset,d12,ratio20,44shards,1epoch,wd.42", + "device_type": "", + "fp8": false, + "fp8_recipe": "tensorwise", + "depth": 12, + "aspect_ratio": 64, + "head_dim": 128, + "max_seq_len": 2048, + "window_pattern": "L", + "num_iterations": -1, + "target_flops": -1.0, + "target_param_data_ratio": 20.0, + "device_batch_size": 16, + "total_batch_size": 524288, + "embedding_lr": 0.3, + "unembedding_lr": 0.008, + "weight_decay": 0.42, + "matrix_lr": 0.02, + "scalar_lr": 0.5, + "warmup_steps": 40, + "warmdown_ratio": 0.65, + "final_lr_frac": 0.05, + "resume_from_step": -1, + "pretokenized": true, + "data_dir": "/content/nanochat_cache/experiments/think-d12-1ep-44sh-r20-wd42/data", + "tokenizer_dir": "/content/nanochat_cache/experiments/think-d12-1ep-44sh-r20-wd42/tokenizer", + "pretokenized_dir": "/content/nanochat_cache/experiments/think-d12-1ep-44sh-r20-wd42/pretok", + "checkpoint_dir": "/content/nanochat_cache/experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints", + "experiment_id": "think-d12-1ep-44sh-r20-wd42", + "experiment_config": "/content/nanochat_cache/experiments/think-d12-1ep-44sh-r20-wd42/config.json", + "tokenizer_fingerprint": "1744d7b7ee0d5d80", + "git_commit_sha": "fd4d149a650d0fb07ae9b96864bb40e5769622d0", + "seed": 42, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "core_metric_max_per_task": 500, + "sample_every": -1, + "save_every": 500, + "model_tag": "think-d12-1ep-44sh-r20-wd42", + "experiment": { + "schema_version": 1, + "stage": "base", + "experiment_id": "think-d12-1ep-44sh-r20-wd42", + "dataset": { + "adapter": "parquet_shards", + "repo": "jbduran/think-dataset", + "revision": "main", + "validation_shard": 472, + "num_train_shards": 44, + "download_workers": 4 + }, + "tokenizer": { + "mode": "train", + "max_chars": 2000000000, + "doc_cap": 10000, + "vocab_size": 32768 + }, + "pretokenize": { + "enabled": true, + "slack": 1.03, + "val_tokens": 20971520, + "shard_tokens": 100000000, + "tokenizer_threads": 8 + }, + "training": { + "depth": 12, + "scaling_params": 110100912, + "target_param_data_ratio": 20.0, + "window_pattern": "L", + "weight_decay": 0.42, + "device_batch_size": 16, + "total_batch_size": 524288, + "save_every": 500, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "sample_every": -1 + }, + "artifacts": { + "repo": "jbduran/think.nano" + }, + "wandb": { + "entity": "jbduran-thinkingmachinesncsu", + "project": "think.nano", + "name": "think-d12-1ep-44sh-r20-wd42", + "group": "think-d12", + "tags": [ + "think-dataset", + "d12", + "ratio20", + "44shards", + "1epoch", + "wd.42" + ] + }, + "config_fingerprint": "a8d632854c2cd1bd", + "artifact_path": "experiments/think-d12-1ep-44sh-r20-wd42" + }, + "stage": "base", + "base_experiment_id": "think-d12-1ep-44sh-r20-wd42", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "config_fingerprint": "a8d632854c2cd1bd" + }, + "device_batch_size": 16, + "max_seq_len": 2048, + "total_batch_size": 524288, + "dataloader_state_dict": { + "file_idx": 22, + "pos": 2109569, + "epoch": 1, + "pq_idx": 22, + "rg_idx": 2109569 + }, + "loop_state": { + "min_val_bpb": 1.078128321044417, + "smooth_train_loss": 2.8475676426206853, + "total_training_time": 11219.814347743988, + "stage_training_flops": 1953398065515724800, + "inherited_parent_flops": 0.0, + "cumulative_pipeline_training_flops": 1953398065515724800 + } +} \ No newline at end of file diff --git a/experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints/model_000500.pt b/experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints/model_000500.pt new file mode 100644 index 0000000000000000000000000000000000000000..169cfe116af1b113e56004f1e61f4151ae75537f --- /dev/null +++ b/experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints/model_000500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:db546864ca01fbe6875d9c12a049bf0aa4232ba3a4dbdba03ecbdd1095a15e55 +size 792761690 diff --git a/experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints/model_001000.pt b/experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints/model_001000.pt new file mode 100644 index 0000000000000000000000000000000000000000..8970bffdd59cd1eeb8f8c0e92b26bbb795ef87ba --- /dev/null +++ b/experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints/model_001000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5af8aecdefdbf085e32749ea53e8ce55622196b42c90634282458c34af4d5e1a +size 792761690 diff --git a/experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints/model_001500.pt b/experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints/model_001500.pt new file mode 100644 index 0000000000000000000000000000000000000000..7cf46a2b216866d8eb3279f23c0f691b16fb313d --- /dev/null +++ b/experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints/model_001500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b7582f0335e8a47d987ffb85174e35b14fdb939ea858448b32e21c56d3384056 +size 792761690 diff --git a/experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints/model_002000.pt b/experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints/model_002000.pt new file mode 100644 index 0000000000000000000000000000000000000000..66fd446a907daec718a8277446503de47ab352c3 --- /dev/null +++ b/experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints/model_002000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c8ead5070dd054e0ea8f29381274296936bedcbc712ffd4fd64b11adbda9e088 +size 792761690 diff --git a/experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints/model_002500.pt b/experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints/model_002500.pt new file mode 100644 index 0000000000000000000000000000000000000000..b98f0b247fe4a9b1ee650d370525abb920e51265 --- /dev/null +++ b/experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints/model_002500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:77f5679e681a78bf061e9cf261a005a4c658b107e974be777f85e489d8ad1140 +size 792761690 diff --git a/experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints/model_003000.pt b/experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints/model_003000.pt new file mode 100644 index 0000000000000000000000000000000000000000..f2378febb625e9cb71754ab214236320def49fe4 --- /dev/null +++ b/experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints/model_003000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ca8be32656940ecf969903f3b513a3e673c74e5e405322c584f57e85a13df0f +size 792761690 diff --git a/experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints/model_003500.pt b/experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints/model_003500.pt new file mode 100644 index 0000000000000000000000000000000000000000..ffa2121a48300e02c9db6372a5b746c52fd89c9c --- /dev/null +++ b/experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints/model_003500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b330320b63a95624ecc9ba05722dda9609593987c48a6ab90047c94c9f3d06f1 +size 792761690 diff --git a/experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints/model_004000.pt b/experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints/model_004000.pt new file mode 100644 index 0000000000000000000000000000000000000000..32073807e5fa8af036bf75f0ed2a635bf315de37 --- /dev/null +++ b/experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints/model_004000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b7986875a99712f761a3f218dfefbca73547a1ae8b45f08ab9d95c7dec1b2a3f +size 792761690 diff --git a/experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints/model_004200.pt b/experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints/model_004200.pt new file mode 100644 index 0000000000000000000000000000000000000000..cec3bdf5fb68d41c36445e868ca6d34f05b4b472 --- /dev/null +++ b/experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints/model_004200.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d641bed4b7bf0e787b469394a0c01c73d1d66655be72525a4c625e3f04c961a6 +size 792761690 diff --git a/experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints/optim_000500_rank0.pt b/experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints/optim_000500_rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..1c7073c47e64bfcaa6f5da191c51485a5d5ab7a5 --- /dev/null +++ b/experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints/optim_000500_rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1525cb06d3f15e13db9115c5bb67d4ba9c751c6c2bc2a5901d615a785323bd3f +size 1246165357 diff --git a/experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints/optim_001000_rank0.pt b/experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints/optim_001000_rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..e563a9b78302e96c954592f625012067e807c95a --- /dev/null +++ b/experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints/optim_001000_rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1181d48ea007611efdd9c45275aa6cbc350d176bf736aef963b05953f4e2837f +size 1246165357 diff --git a/experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints/optim_001500_rank0.pt b/experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints/optim_001500_rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..8d78655938b9a98d8e34e6e6c9ba10e889f82fcc --- /dev/null +++ b/experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints/optim_001500_rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:93fcab3d3b289e92644d578ebcc8b9ea9dd644fe3e36efd4ebda802e0f438b42 +size 1246165357 diff --git a/experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints/optim_002000_rank0.pt b/experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints/optim_002000_rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..ec5083fdf7f2c48abf607333b9a1cad50e925df1 --- /dev/null +++ b/experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints/optim_002000_rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6602307370d1412fd6f58d5e1b67da2edd22590385d684ec62d1640075409d99 +size 1246165357 diff --git a/experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints/optim_002500_rank0.pt b/experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints/optim_002500_rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..1993a11840496c591652fcd3e05a7eef136477c7 --- /dev/null +++ b/experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints/optim_002500_rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fde77b3022a5126a82a110fb379c4aec5775aa6440623ea42a6790c99a721422 +size 1246165357 diff --git a/experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints/optim_003000_rank0.pt b/experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints/optim_003000_rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..17da371afdca0ab56fa2c3f7051e3bc1bd87cb2c --- /dev/null +++ b/experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints/optim_003000_rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c081ca79e912393a33cf8e63d1795047a87d90735757f51df8ed38477ba851d5 +size 1246165357 diff --git a/experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints/optim_003500_rank0.pt b/experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints/optim_003500_rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..8580d78e2d37cf857ed203cb500371b6bef7df72 --- /dev/null +++ b/experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints/optim_003500_rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:328e0b85a4407f35d10e644574bad923147a25c24dfbc5e8ebf0b095c2424497 +size 1246165357 diff --git a/experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints/optim_004000_rank0.pt b/experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints/optim_004000_rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..b17f70cc46f019e2eb03de9327732f713f34901e --- /dev/null +++ b/experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints/optim_004000_rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:704c4809a117c46020fcc551cfef13bd6aa5259a74ce8ab36acfd9993b126436 +size 1246165357 diff --git a/experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints/optim_004200_rank0.pt b/experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints/optim_004200_rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..fa3c696227f942e24fcde49d4fe324c97b7dac97 --- /dev/null +++ b/experiments/think-d12-1ep-44sh-r20-wd42/base_checkpoints/optim_004200_rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8c8ff3203701457409f4a8b416d178fb66b3ced9764bda87bccf48e0c1990cb6 +size 1246165357 diff --git a/experiments/think-d12-1ep-44sh-r20-wd42/config.json b/experiments/think-d12-1ep-44sh-r20-wd42/config.json new file mode 100644 index 0000000000000000000000000000000000000000..e87fa0fbd8c1e8dbf89b141b813891483f72d194 --- /dev/null +++ b/experiments/think-d12-1ep-44sh-r20-wd42/config.json @@ -0,0 +1,59 @@ +{ + "schema_version": 1, + "stage": "base", + "experiment_id": "think-d12-1ep-44sh-r20-wd42", + "dataset": { + "adapter": "parquet_shards", + "repo": "jbduran/think-dataset", + "revision": "main", + "validation_shard": 472, + "num_train_shards": 44, + "download_workers": 4 + }, + "tokenizer": { + "mode": "train", + "max_chars": 2000000000, + "doc_cap": 10000, + "vocab_size": 32768 + }, + "pretokenize": { + "enabled": true, + "slack": 1.03, + "val_tokens": 20971520, + "shard_tokens": 100000000, + "tokenizer_threads": 8 + }, + "training": { + "depth": 12, + "scaling_params": 110100912, + "target_param_data_ratio": 20.0, + "window_pattern": "L", + "weight_decay": 0.42, + "device_batch_size": 16, + "total_batch_size": 524288, + "save_every": 500, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "sample_every": -1 + }, + "artifacts": { + "repo": "jbduran/think.nano" + }, + "wandb": { + "entity": "jbduran-thinkingmachinesncsu", + "project": "think.nano", + "name": "think-d12-1ep-44sh-r20-wd42", + "group": "think-d12", + "tags": [ + "think-dataset", + "d12", + "ratio20", + "44shards", + "1epoch", + "wd.42" + ] + }, + "config_fingerprint": "a8d632854c2cd1bd", + "artifact_path": "experiments/think-d12-1ep-44sh-r20-wd42" +} diff --git a/experiments/think-d12-1ep-44sh-r20-wd42/evals/core.json b/experiments/think-d12-1ep-44sh-r20-wd42/evals/core.json new file mode 100644 index 0000000000000000000000000000000000000000..baa7f31846ea732ba51513ea0abd5f9215d4b752 --- /dev/null +++ b/experiments/think-d12-1ep-44sh-r20-wd42/evals/core.json @@ -0,0 +1,56 @@ +{ + "model": "base_model (step 4200)", + "step": 4200, + "bpb": {}, + "core_metric": 0.07908813484502109, + "core_results": { + "hellaswag_zeroshot": 0.28251343965530396, + "jeopardy": 0.0009447330958209932, + "bigbench_qa_wikidata": 0.07814575731754303, + "arc_easy": 0.31607744097709656, + "arc_challenge": 0.2022184282541275, + "copa": 0.5699999928474426, + "commonsense_qa": 0.312039315700531, + "piqa": 0.5527747273445129, + "openbook_qa": 0.24800001084804535, + "lambada_openai": 0.26043081283569336, + "hellaswag": 0.2815176248550415, + "winograd": 0.5604395866394043, + "winogrande": 0.4980268180370331, + "bigbench_dyck_languages": 0.11500000208616257, + "agi_eval_lsat_ar": 0.260869562625885, + "bigbench_cs_algorithms": 0.4015151262283325, + "bigbench_operators": 0.10476190596818924, + "bigbench_repeat_copy_logic": 0.0, + "squad": 0.025922421365976334, + "coqa": 0.0899411216378212, + "boolq": 0.542201817035675, + "bigbench_language_identification": 0.25669997930526733 + }, + "centered_results": { + "hellaswag_zeroshot": 0.04335125287373861, + "jeopardy": 0.0009447330958209932, + "bigbench_qa_wikidata": 0.07814575731754303, + "arc_easy": 0.08810325463612874, + "arc_challenge": -0.06370876232783, + "copa": 0.13999998569488525, + "commonsense_qa": 0.14004914462566373, + "piqa": 0.10554945468902588, + "openbook_qa": -0.002666652202606201, + "lambada_openai": 0.26043081283569336, + "hellaswag": 0.04202349980672201, + "winograd": 0.1208791732788086, + "winogrande": -0.003946363925933838, + "bigbench_dyck_languages": 0.11500000208616257, + "agi_eval_lsat_ar": 0.07608695328235625, + "bigbench_cs_algorithms": 0.4015151262283325, + "bigbench_operators": 0.10476190596818924, + "bigbench_repeat_copy_logic": 0.0, + "squad": 0.025922421365976334, + "coqa": 0.0899411216378212, + "boolq": -0.20473206043243405, + "bigbench_language_identification": 0.1822882060563997 + }, + "conditioned_samples": [], + "unconditioned_samples": [] +} \ No newline at end of file diff --git a/experiments/think-d12-1ep-44sh-r20-wd42/evals/samples.json b/experiments/think-d12-1ep-44sh-r20-wd42/evals/samples.json new file mode 100644 index 0000000000000000000000000000000000000000..24c18830774960bc64d77adec7f2ffeaa2c81052 --- /dev/null +++ b/experiments/think-d12-1ep-44sh-r20-wd42/evals/samples.json @@ -0,0 +1,48 @@ +{ + "model": "base_model (step 4200)", + "step": 4200, + "bpb": {}, + "core_metric": null, + "core_results": null, + "centered_results": null, + "conditioned_samples": [ + { + "prompt": "The capital of France is", + "text": "<|bos|>The capital of France is the capital of the world. \n\nThe capital of the world is the capital of" + }, + { + "prompt": "The chemical symbol of gold is", + "text": "<|bos|>The chemical symbol of gold is the symbol of the gold of the world. The gold of the world is the" + }, + { + "prompt": "If yesterday was Friday, then tomorrow will be", + "text": "<|bos|>If yesterday was Friday, then tomorrow will be the last day of the week. \n\nI am, dear Sir, your most" + }, + { + "prompt": "The opposite of hot is", + "text": "<|bos|>The opposite of hot is the best of all. \n\nThe best of all is the best of all." + }, + { + "prompt": "The planets of the solar system are:", + "text": "<|bos|>The planets of the solar system are: \n\n1. The sun, which is the sun of the solar system. \n\n" + }, + { + "prompt": "My favorite color is", + "text": "<|bos|>My favorite color is the same as that of the sun, and the same as that of the moon" + }, + { + "prompt": "If 5*x + 3 = 13, then x is", + "text": "<|bos|>If 5*x + 3 = 13, then x is the number of the first and second, and x is the number of the second" + } + ], + "unconditioned_samples": [ + "<|bos|>IT\n\nS. PERMISSION Dedicated to the CSTRORRS OF BOSTON. \n\nADVERTISEMENT \n\nWHILE I am following up the translation of the \"Saul,\" it will not be useless to point out to the different subscribers, and to each subscriber, the different phases, phases and phases of the phenomena presented - an appearance presented in all its phases and genera, and common to them all, so as to convey to all minds then present, an impression which is correct as it is true. \n\nYet a work is what it professes to be, and this, to a very considerable extent, ought to go on for ever", + "<|bos|>900 women, or 96,000 or \n\n105,000 inhabitants, and in general the Morningags. Their occupation was not pleasant. . Some of them were married, and had children. Of these the \n\nMorningags were the principal; the inferior were careful at school. \n\nTheir style is not unlike that of the Turkish potter. Here the grand aim is to obtain a common arena. . . . . To English readers of books, the present day is one of the most exciting and disappointing experiences in European literature. We carelessly allow this people to be the exclusive champions of civilization and true civilization.", + "<|bos|>NY besides that and never' 't can on' weighing ourselves. To be dead'st... . . or dreaming'st......\n\nOur'ret too comin' (2) to the ordinary, non.. \n\nFloors (1) to pleasoit person. \n\nSane (a) can amply and gratuitously as long-a.. \n\nOn' puntity, be more alive... \n\nWaiving, too secundity, let us (y) take... \n\nAre cases ofctority sufficient, sometimes, epis-ty, disELLY \n\nORKILL idea. \n\nPUR-BO", + "<|bos|>-boat built by John Eddy \n\nGreen... 623 \n\nJEFFERSON, JOHN (b. Jan. 1831), commenced business as hotel and tavern-keeper at \n\nFickenkamp, Cal., Nov. 26, 1827.. \n\n289; succeeded to business as hotel-keeper and proprietor Dec. 22, 1844.. \n\n320; commenced his business as hotel-keeper and thenceforth became a hotel and boarding-house keeper. \n\n323; successfully carried on business as hotel-keeper and thenceforth became a hotel and boarding-house keeper.. \n\n329; at end of ", + "<|bos|>. \n\nColored by Hugh Angola M'Nabbs, Commodore James E. Lightwood, Notables. This in- amidships. ventilation of the service contests in the testingroom of the national cemetery will facilitate the work among sailors who are anxious to see their fellow-patriots die.\n\nColored from a Painting, by Luella Vancouver, L.S. \n\nNoticed by Asa G-Giveno. Feather.\n\nThere are grave dangers to the hospital which must be avoided.\n\nAcres of Described by Charles. \n\nSir Francis Drake's Louisiana-Book, $ 1555-1571", + "<|bos|>. \n\n TRUSTEES.]. [The whole difference between the trust companies of his farm, Blodgett v. Nugent, 63 K. B. 521, and the trust companies of Reingeldt v. Wilbraham, 95 A. 118, 56 Am. St. Rep. 232, was merely a clear difference of intention.]\n\nIt was also clear there had been a necessary delivery of the sound overseas cart can Turnusey v. Metropolitan St., etc., R. Co., 58 L. R. A. 655, and was a clear and acquies", + "<|bos|>ING Cosmopolites. \n\nSurviving Evidence. \n\nAs to the main objects of the girl's introduction, not necessarily to a rendition of 820-44 22305 N229 Jones concludes that a report of 22-8 has been inserted in this case regarding]\n\nburying be given by me, and that it was thought that this would facilitate our future proceedings.\n\nSome attempt was made in vain to find the advertisement notificatione by defendant, but one defendant named as gruffly as the was, namely, the boy M\u00e1 vhdvr\u00e1 to groom of the horse;", + "<|bos|> gilt west, sundry small pieces of paper were found in his apartment.\n\n1762.] buoyant as lightning. This piece, preserved in a drawer in the library of the British House of Commons, was a very bad article, and not a few of the proprietor's horse fell off as it fell from him. His mother was crying, and a number of other women were taking care of their milk boxes under the bed-window. One of them stripped the stranger of his best clothes. After the fireman had been succeeded to some small articles which were every moment received with peculiar satisfaction by his buttons and clenched f" + ] +} \ No newline at end of file diff --git a/experiments/think-d12-1ep-44sh-r20-wd42/evals/val_bpb.json b/experiments/think-d12-1ep-44sh-r20-wd42/evals/val_bpb.json new file mode 100644 index 0000000000000000000000000000000000000000..ce33926da84abd0a27d2ed8d5755a60b19aefc80 --- /dev/null +++ b/experiments/think-d12-1ep-44sh-r20-wd42/evals/val_bpb.json @@ -0,0 +1,12 @@ +{ + "model": "base_model (step 4200)", + "step": 4200, + "bpb": { + "val": 1.0191548981297465 + }, + "core_metric": null, + "core_results": null, + "centered_results": null, + "conditioned_samples": [], + "unconditioned_samples": [] +} \ No newline at end of file diff --git a/experiments/think-d12-1ep-44sh-r20-wd42/run.json b/experiments/think-d12-1ep-44sh-r20-wd42/run.json new file mode 100644 index 0000000000000000000000000000000000000000..990617f6764d3b22ee9d07d0161630ea14804f40 --- /dev/null +++ b/experiments/think-d12-1ep-44sh-r20-wd42/run.json @@ -0,0 +1,10 @@ +{ + "experiment_id": "think-d12-1ep-44sh-r20-wd42", + "stage": "base", + "base_experiment_id": "think-d12-1ep-44sh-r20-wd42", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "config_fingerprint": "a8d632854c2cd1bd", + "wandb_run_id": "5c4fba8a", + "created_at": 1781881918 +} diff --git a/experiments/think-d12-1ep-44sh-r20-wd42/summary.json b/experiments/think-d12-1ep-44sh-r20-wd42/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..ea775df32bee7a4e334fa46d42d2dd8a5b054afc --- /dev/null +++ b/experiments/think-d12-1ep-44sh-r20-wd42/summary.json @@ -0,0 +1,93 @@ +{ + "experiment_id": "think-d12-1ep-44sh-r20-wd42", + "stage": "base", + "base_experiment_id": "think-d12-1ep-44sh-r20-wd42", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "dataset": "jbduran/think-dataset", + "dataset_revision": "main", + "step": 4200, + "depth": 12, + "target_param_data_ratio": 20.0, + "training_tokens": 2202009600, + "final_sampled_val_bpb": 1.078128321044417, + "minimum_sampled_val_bpb": 1.078128321044417, + "full_val_bpb": 1.0191548981297465, + "core_metric": 0.07908813484502109, + "centered_results": { + "hellaswag_zeroshot": 0.04335125287373861, + "jeopardy": 0.0009447330958209932, + "bigbench_qa_wikidata": 0.07814575731754303, + "arc_easy": 0.08810325463612874, + "arc_challenge": -0.06370876232783, + "copa": 0.13999998569488525, + "commonsense_qa": 0.14004914462566373, + "piqa": 0.10554945468902588, + "openbook_qa": -0.002666652202606201, + "lambada_openai": 0.26043081283569336, + "hellaswag": 0.04202349980672201, + "winograd": 0.1208791732788086, + "winogrande": -0.003946363925933838, + "bigbench_dyck_languages": 0.11500000208616257, + "agi_eval_lsat_ar": 0.07608695328235625, + "bigbench_cs_algorithms": 0.4015151262283325, + "bigbench_operators": 0.10476190596818924, + "bigbench_repeat_copy_logic": 0.0, + "squad": 0.025922421365976334, + "coqa": 0.0899411216378212, + "boolq": -0.20473206043243405, + "bigbench_language_identification": 0.1822882060563997 + }, + "conditioned_samples": [ + { + "prompt": "The capital of France is", + "text": "<|bos|>The capital of France is the capital of the world. \n\nThe capital of the world is the capital of" + }, + { + "prompt": "The chemical symbol of gold is", + "text": "<|bos|>The chemical symbol of gold is the symbol of the gold of the world. The gold of the world is the" + }, + { + "prompt": "If yesterday was Friday, then tomorrow will be", + "text": "<|bos|>If yesterday was Friday, then tomorrow will be the last day of the week. \n\nI am, dear Sir, your most" + }, + { + "prompt": "The opposite of hot is", + "text": "<|bos|>The opposite of hot is the best of all. \n\nThe best of all is the best of all." + }, + { + "prompt": "The planets of the solar system are:", + "text": "<|bos|>The planets of the solar system are: \n\n1. The sun, which is the sun of the solar system. \n\n" + }, + { + "prompt": "My favorite color is", + "text": "<|bos|>My favorite color is the same as that of the sun, and the same as that of the moon" + }, + { + "prompt": "If 5*x + 3 = 13, then x is", + "text": "<|bos|>If 5*x + 3 = 13, then x is the number of the first and second, and x is the number of the second" + } + ], + "unconditioned_samples": [ + "<|bos|>IT\n\nS. PERMISSION Dedicated to the CSTRORRS OF BOSTON. \n\nADVERTISEMENT \n\nWHILE I am following up the translation of the \"Saul,\" it will not be useless to point out to the different subscribers, and to each subscriber, the different phases, phases and phases of the phenomena presented - an appearance presented in all its phases and genera, and common to them all, so as to convey to all minds then present, an impression which is correct as it is true. \n\nYet a work is what it professes to be, and this, to a very considerable extent, ought to go on for ever", + "<|bos|>900 women, or 96,000 or \n\n105,000 inhabitants, and in general the Morningags. Their occupation was not pleasant. . Some of them were married, and had children. Of these the \n\nMorningags were the principal; the inferior were careful at school. \n\nTheir style is not unlike that of the Turkish potter. Here the grand aim is to obtain a common arena. . . . . To English readers of books, the present day is one of the most exciting and disappointing experiences in European literature. We carelessly allow this people to be the exclusive champions of civilization and true civilization.", + "<|bos|>NY besides that and never' 't can on' weighing ourselves. To be dead'st... . . or dreaming'st......\n\nOur'ret too comin' (2) to the ordinary, non.. \n\nFloors (1) to pleasoit person. \n\nSane (a) can amply and gratuitously as long-a.. \n\nOn' puntity, be more alive... \n\nWaiving, too secundity, let us (y) take... \n\nAre cases ofctority sufficient, sometimes, epis-ty, disELLY \n\nORKILL idea. \n\nPUR-BO", + "<|bos|>-boat built by John Eddy \n\nGreen... 623 \n\nJEFFERSON, JOHN (b. Jan. 1831), commenced business as hotel and tavern-keeper at \n\nFickenkamp, Cal., Nov. 26, 1827.. \n\n289; succeeded to business as hotel-keeper and proprietor Dec. 22, 1844.. \n\n320; commenced his business as hotel-keeper and thenceforth became a hotel and boarding-house keeper. \n\n323; successfully carried on business as hotel-keeper and thenceforth became a hotel and boarding-house keeper.. \n\n329; at end of ", + "<|bos|>. \n\nColored by Hugh Angola M'Nabbs, Commodore James E. Lightwood, Notables. This in- amidships. ventilation of the service contests in the testingroom of the national cemetery will facilitate the work among sailors who are anxious to see their fellow-patriots die.\n\nColored from a Painting, by Luella Vancouver, L.S. \n\nNoticed by Asa G-Giveno. Feather.\n\nThere are grave dangers to the hospital which must be avoided.\n\nAcres of Described by Charles. \n\nSir Francis Drake's Louisiana-Book, $ 1555-1571", + "<|bos|>. \n\n TRUSTEES.]. [The whole difference between the trust companies of his farm, Blodgett v. Nugent, 63 K. B. 521, and the trust companies of Reingeldt v. Wilbraham, 95 A. 118, 56 Am. St. Rep. 232, was merely a clear difference of intention.]\n\nIt was also clear there had been a necessary delivery of the sound overseas cart can Turnusey v. Metropolitan St., etc., R. Co., 58 L. R. A. 655, and was a clear and acquies", + "<|bos|>ING Cosmopolites. \n\nSurviving Evidence. \n\nAs to the main objects of the girl's introduction, not necessarily to a rendition of 820-44 22305 N229 Jones concludes that a report of 22-8 has been inserted in this case regarding]\n\nburying be given by me, and that it was thought that this would facilitate our future proceedings.\n\nSome attempt was made in vain to find the advertisement notificatione by defendant, but one defendant named as gruffly as the was, namely, the boy M\u00e1 vhdvr\u00e1 to groom of the horse;", + "<|bos|> gilt west, sundry small pieces of paper were found in his apartment.\n\n1762.] buoyant as lightning. This piece, preserved in a drawer in the library of the British House of Commons, was a very bad article, and not a few of the proprietor's horse fell off as it fell from him. His mother was crying, and a number of other women were taking care of their milk boxes under the bed-window. One of them stripped the stranger of his best clothes. After the fireman had been succeeded to some small articles which were every moment received with peculiar satisfaction by his buttons and clenched f" + ], + "training_time_seconds": 11219.814347743988, + "stage_training_flops": 1.9533980655157248e+18, + "inherited_parent_flops": 0.0, + "cumulative_pipeline_training_flops": 1.9533980655157248e+18, + "config_fingerprint": "a8d632854c2cd1bd", + "git_commit_sha": "fd4d149a650d0fb07ae9b96864bb40e5769622d0", + "wandb_url": "https://wandb.ai/jbduran-thinkingmachinesncsu/think.nano/runs/5c4fba8a", + "huggingface_url": "https://huggingface.co/jbduran/think.nano/tree/main/experiments/think-d12-1ep-44sh-r20-wd42", + "dataset_fingerprint": "a6e1b3a100e0d8b3", + "tokenizer_fingerprint": "1744d7b7ee0d5d80", + "unique_train_tokens": 2268069888, + "effective_epochs": 0.970873786407767 +} diff --git a/experiments/think-d12-1ep-44sh-r20-wd42/tokenizer/experiment_tokenizer.json b/experiments/think-d12-1ep-44sh-r20-wd42/tokenizer/experiment_tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..e4983b47a9e5fee5f46c909e962afe2310ae1dbe --- /dev/null +++ b/experiments/think-d12-1ep-44sh-r20-wd42/tokenizer/experiment_tokenizer.json @@ -0,0 +1,18 @@ +{ + "experiment_id": "think-d12-1ep-44sh-r20-wd42", + "dataset": { + "adapter": "parquet_shards", + "repo": "jbduran/think-dataset", + "revision": "main", + "validation_shard": 472, + "num_train_shards": 44, + "download_workers": 4 + }, + "tokenizer": { + "mode": "train", + "max_chars": 2000000000, + "doc_cap": 10000, + "vocab_size": 32768 + }, + "created_at": 1781881934 +} diff --git a/experiments/think-d12-1ep-44sh-r20-wd42/tokenizer/token_bytes.pt b/experiments/think-d12-1ep-44sh-r20-wd42/tokenizer/token_bytes.pt new file mode 100644 index 0000000000000000000000000000000000000000..80dbb386d071538021ab399ee6e965ae0cd1a54e --- /dev/null +++ b/experiments/think-d12-1ep-44sh-r20-wd42/tokenizer/token_bytes.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:59f928e04aa2ac37dd4064493240d1e73ecab7acb217c5a183311b0c523a3468 +size 132649 diff --git a/experiments/think-d12-1ep-44sh-r20-wd42/tokenizer/tokenizer.pkl b/experiments/think-d12-1ep-44sh-r20-wd42/tokenizer/tokenizer.pkl new file mode 100644 index 0000000000000000000000000000000000000000..a812730c3de1acc1e9e30ef6c4ccf1e9360f5d8a --- /dev/null +++ b/experiments/think-d12-1ep-44sh-r20-wd42/tokenizer/tokenizer.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fd583e3c35851c62295a1a0f6d688923e4f30649ac963b701ec2440fe8bc3e4f +size 404221 diff --git a/experiments/think-d12-1ep-65sh-r30/base_checkpoints/meta_000500.json b/experiments/think-d12-1ep-65sh-r30/base_checkpoints/meta_000500.json new file mode 100644 index 0000000000000000000000000000000000000000..350bf0bb1e52e063da5adc52977e79f661b50c44 --- /dev/null +++ b/experiments/think-d12-1ep-65sh-r30/base_checkpoints/meta_000500.json @@ -0,0 +1,155 @@ +{ + "step": 500, + "experiment_id": "think-d12-1ep-65sh-r30", + "val_bpb": 1.323730996039066, + "model_config": { + "sequence_len": 2048, + "vocab_size": 32768, + "n_layer": 12, + "n_head": 6, + "n_kv_head": 6, + "n_embd": 768, + "window_pattern": "L" + }, + "user_config": { + "run": "think-d12-1ep-65sh-r30", + "wandb_run_id": "6465e19b", + "wandb_group": "think-d12-stopping-point", + "wandb_tags": "think-dataset,d12,ratio30,65-shards,a100,bf16,stopping-point", + "device_type": "cuda", + "fp8": false, + "fp8_recipe": "tensorwise", + "depth": 12, + "aspect_ratio": 64, + "head_dim": 128, + "max_seq_len": 2048, + "window_pattern": "L", + "num_iterations": -1, + "target_flops": -1.0, + "target_param_data_ratio": 30.0, + "device_batch_size": 16, + "total_batch_size": 524288, + "embedding_lr": 0.3, + "unembedding_lr": 0.008, + "weight_decay": 0.28, + "matrix_lr": 0.02, + "scalar_lr": 0.5, + "warmup_steps": 40, + "warmdown_ratio": 0.65, + "final_lr_frac": 0.05, + "resume_from_step": -1, + "pretokenized": true, + "data_dir": "/content/nanochat_cache/experiments/think-d12-1ep-65sh-r30/data", + "tokenizer_dir": "/content/nanochat_cache/experiments/think-d12-1ep-65sh-r30/tokenizer", + "pretokenized_dir": "/content/nanochat_cache/experiments/think-d12-1ep-65sh-r30/pretok", + "checkpoint_dir": "/content/nanochat_cache/experiments/think-d12-1ep-65sh-r30/base_checkpoints", + "experiment_id": "think-d12-1ep-65sh-r30", + "experiment_config": "/content/nanochat_cache/experiments/think-d12-1ep-65sh-r30/config.json", + "tokenizer_fingerprint": "db3bec0946e70097", + "git_commit_sha": "09b5f3b7b61a45f34d50a36e570b85f0259208bc", + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": 500, + "core_metric_max_per_task": 50, + "sample_every": -1, + "save_every": 500, + "model_tag": "think-d12-1ep-65sh-r30", + "experiment": { + "schema_version": 1, + "stage": "base", + "experiment_id": "think-d12-1ep-65sh-r30", + "dataset": { + "adapter": "parquet_shards", + "repo": "jbduran/think-dataset", + "revision": "main", + "validation_shard": 472, + "num_train_shards": 65, + "download_workers": 4 + }, + "tokenizer": { + "mode": "train", + "max_chars": 2000000000, + "doc_cap": 10000, + "vocab_size": 32768 + }, + "pretokenize": { + "enabled": true, + "target_tokens": 3402104832, + "slack": 1.03, + "require_no_wrap": true, + "val_tokens": 20971520, + "shard_tokens": 100000000, + "tokenizer_threads": 8 + }, + "training": { + "device_type": "cuda", + "depth": 12, + "aspect_ratio": 64, + "head_dim": 128, + "max_seq_len": 2048, + "scaling_params": 110100912, + "target_param_data_ratio": 30.0, + "window_pattern": "L", + "device_batch_size": 16, + "total_batch_size": 524288, + "embedding_lr": 0.3, + "unembedding_lr": 0.008, + "weight_decay": 0.28, + "matrix_lr": 0.02, + "scalar_lr": 0.5, + "warmup_steps": 40, + "warmdown_ratio": 0.65, + "final_lr_frac": 0.05, + "save_every": 500, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": 500, + "core_metric_max_per_task": 50, + "sample_every": -1 + }, + "artifacts": { + "repo": "jbduran/think.nano" + }, + "wandb": { + "entity": "jbduran-thinkingmachinesncsu", + "project": "think.nano", + "name": "think-d12-1ep-65sh-r30", + "group": "think-d12-stopping-point", + "tags": [ + "think-dataset", + "d12", + "ratio30", + "65-shards", + "a100", + "bf16", + "stopping-point" + ] + }, + "config_fingerprint": "35996219a51996ca", + "artifact_path": "experiments/think-d12-1ep-65sh-r30" + }, + "stage": "base", + "base_experiment_id": "think-d12-1ep-65sh-r30", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "config_fingerprint": "35996219a51996ca" + }, + "device_batch_size": 16, + "max_seq_len": 2048, + "total_batch_size": 524288, + "dataloader_state_dict": { + "file_idx": 2, + "pos": 62184769, + "epoch": 1, + "pq_idx": 2, + "rg_idx": 62184769 + }, + "loop_state": { + "min_val_bpb": 1.323730996039066, + "smooth_train_loss": 3.6702374931405064, + "total_training_time": 1313.3025135993958, + "stage_training_flops": 232547388751872000, + "inherited_parent_flops": 0.0, + "cumulative_pipeline_training_flops": 232547388751872000 + } +} \ No newline at end of file diff --git a/experiments/think-d12-1ep-65sh-r30/base_checkpoints/meta_001000.json b/experiments/think-d12-1ep-65sh-r30/base_checkpoints/meta_001000.json new file mode 100644 index 0000000000000000000000000000000000000000..3a2f6410d80646d561db3ebc633aad6cc00f3a36 --- /dev/null +++ b/experiments/think-d12-1ep-65sh-r30/base_checkpoints/meta_001000.json @@ -0,0 +1,155 @@ +{ + "step": 1000, + "experiment_id": "think-d12-1ep-65sh-r30", + "val_bpb": 1.253497672143533, + "model_config": { + "sequence_len": 2048, + "vocab_size": 32768, + "n_layer": 12, + "n_head": 6, + "n_kv_head": 6, + "n_embd": 768, + "window_pattern": "L" + }, + "user_config": { + "run": "think-d12-1ep-65sh-r30", + "wandb_run_id": "6465e19b", + "wandb_group": "think-d12-stopping-point", + "wandb_tags": "think-dataset,d12,ratio30,65-shards,a100,bf16,stopping-point", + "device_type": "cuda", + "fp8": false, + "fp8_recipe": "tensorwise", + "depth": 12, + "aspect_ratio": 64, + "head_dim": 128, + "max_seq_len": 2048, + "window_pattern": "L", + "num_iterations": -1, + "target_flops": -1.0, + "target_param_data_ratio": 30.0, + "device_batch_size": 16, + "total_batch_size": 524288, + "embedding_lr": 0.3, + "unembedding_lr": 0.008, + "weight_decay": 0.28, + "matrix_lr": 0.02, + "scalar_lr": 0.5, + "warmup_steps": 40, + "warmdown_ratio": 0.65, + "final_lr_frac": 0.05, + "resume_from_step": -1, + "pretokenized": true, + "data_dir": "/content/nanochat_cache/experiments/think-d12-1ep-65sh-r30/data", + "tokenizer_dir": "/content/nanochat_cache/experiments/think-d12-1ep-65sh-r30/tokenizer", + "pretokenized_dir": "/content/nanochat_cache/experiments/think-d12-1ep-65sh-r30/pretok", + "checkpoint_dir": "/content/nanochat_cache/experiments/think-d12-1ep-65sh-r30/base_checkpoints", + "experiment_id": "think-d12-1ep-65sh-r30", + "experiment_config": "/content/nanochat_cache/experiments/think-d12-1ep-65sh-r30/config.json", + "tokenizer_fingerprint": "db3bec0946e70097", + "git_commit_sha": "09b5f3b7b61a45f34d50a36e570b85f0259208bc", + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": 500, + "core_metric_max_per_task": 50, + "sample_every": -1, + "save_every": 500, + "model_tag": "think-d12-1ep-65sh-r30", + "experiment": { + "schema_version": 1, + "stage": "base", + "experiment_id": "think-d12-1ep-65sh-r30", + "dataset": { + "adapter": "parquet_shards", + "repo": "jbduran/think-dataset", + "revision": "main", + "validation_shard": 472, + "num_train_shards": 65, + "download_workers": 4 + }, + "tokenizer": { + "mode": "train", + "max_chars": 2000000000, + "doc_cap": 10000, + "vocab_size": 32768 + }, + "pretokenize": { + "enabled": true, + "target_tokens": 3402104832, + "slack": 1.03, + "require_no_wrap": true, + "val_tokens": 20971520, + "shard_tokens": 100000000, + "tokenizer_threads": 8 + }, + "training": { + "device_type": "cuda", + "depth": 12, + "aspect_ratio": 64, + "head_dim": 128, + "max_seq_len": 2048, + "scaling_params": 110100912, + "target_param_data_ratio": 30.0, + "window_pattern": "L", + "device_batch_size": 16, + "total_batch_size": 524288, + "embedding_lr": 0.3, + "unembedding_lr": 0.008, + "weight_decay": 0.28, + "matrix_lr": 0.02, + "scalar_lr": 0.5, + "warmup_steps": 40, + "warmdown_ratio": 0.65, + "final_lr_frac": 0.05, + "save_every": 500, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": 500, + "core_metric_max_per_task": 50, + "sample_every": -1 + }, + "artifacts": { + "repo": "jbduran/think.nano" + }, + "wandb": { + "entity": "jbduran-thinkingmachinesncsu", + "project": "think.nano", + "name": "think-d12-1ep-65sh-r30", + "group": "think-d12-stopping-point", + "tags": [ + "think-dataset", + "d12", + "ratio30", + "65-shards", + "a100", + "bf16", + "stopping-point" + ] + }, + "config_fingerprint": "35996219a51996ca", + "artifact_path": "experiments/think-d12-1ep-65sh-r30" + }, + "stage": "base", + "base_experiment_id": "think-d12-1ep-65sh-r30", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "config_fingerprint": "35996219a51996ca" + }, + "device_batch_size": 16, + "max_seq_len": 2048, + "total_batch_size": 524288, + "dataloader_state_dict": { + "file_idx": 5, + "pos": 24336769, + "epoch": 1, + "pq_idx": 5, + "rg_idx": 24336769 + }, + "loop_state": { + "min_val_bpb": 1.253497672143533, + "smooth_train_loss": 3.404809871021335, + "total_training_time": 2657.893961429596, + "stage_training_flops": 465094777503744000, + "inherited_parent_flops": 0.0, + "cumulative_pipeline_training_flops": 465094777503744000 + } +} \ No newline at end of file diff --git a/experiments/think-d12-1ep-65sh-r30/base_checkpoints/meta_001500.json b/experiments/think-d12-1ep-65sh-r30/base_checkpoints/meta_001500.json new file mode 100644 index 0000000000000000000000000000000000000000..a712defb624393eb101c8daba5979a88969dc3f7 --- /dev/null +++ b/experiments/think-d12-1ep-65sh-r30/base_checkpoints/meta_001500.json @@ -0,0 +1,155 @@ +{ + "step": 1500, + "experiment_id": "think-d12-1ep-65sh-r30", + "val_bpb": 1.2322366946944483, + "model_config": { + "sequence_len": 2048, + "vocab_size": 32768, + "n_layer": 12, + "n_head": 6, + "n_kv_head": 6, + "n_embd": 768, + "window_pattern": "L" + }, + "user_config": { + "run": "think-d12-1ep-65sh-r30", + "wandb_run_id": "6465e19b", + "wandb_group": "think-d12-stopping-point", + "wandb_tags": "think-dataset,d12,ratio30,65-shards,a100,bf16,stopping-point", + "device_type": "cuda", + "fp8": false, + "fp8_recipe": "tensorwise", + "depth": 12, + "aspect_ratio": 64, + "head_dim": 128, + "max_seq_len": 2048, + "window_pattern": "L", + "num_iterations": -1, + "target_flops": -1.0, + "target_param_data_ratio": 30.0, + "device_batch_size": 16, + "total_batch_size": 524288, + "embedding_lr": 0.3, + "unembedding_lr": 0.008, + "weight_decay": 0.28, + "matrix_lr": 0.02, + "scalar_lr": 0.5, + "warmup_steps": 40, + "warmdown_ratio": 0.65, + "final_lr_frac": 0.05, + "resume_from_step": -1, + "pretokenized": true, + "data_dir": "/content/nanochat_cache/experiments/think-d12-1ep-65sh-r30/data", + "tokenizer_dir": "/content/nanochat_cache/experiments/think-d12-1ep-65sh-r30/tokenizer", + "pretokenized_dir": "/content/nanochat_cache/experiments/think-d12-1ep-65sh-r30/pretok", + "checkpoint_dir": "/content/nanochat_cache/experiments/think-d12-1ep-65sh-r30/base_checkpoints", + "experiment_id": "think-d12-1ep-65sh-r30", + "experiment_config": "/content/nanochat_cache/experiments/think-d12-1ep-65sh-r30/config.json", + "tokenizer_fingerprint": "db3bec0946e70097", + "git_commit_sha": "09b5f3b7b61a45f34d50a36e570b85f0259208bc", + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": 500, + "core_metric_max_per_task": 50, + "sample_every": -1, + "save_every": 500, + "model_tag": "think-d12-1ep-65sh-r30", + "experiment": { + "schema_version": 1, + "stage": "base", + "experiment_id": "think-d12-1ep-65sh-r30", + "dataset": { + "adapter": "parquet_shards", + "repo": "jbduran/think-dataset", + "revision": "main", + "validation_shard": 472, + "num_train_shards": 65, + "download_workers": 4 + }, + "tokenizer": { + "mode": "train", + "max_chars": 2000000000, + "doc_cap": 10000, + "vocab_size": 32768 + }, + "pretokenize": { + "enabled": true, + "target_tokens": 3402104832, + "slack": 1.03, + "require_no_wrap": true, + "val_tokens": 20971520, + "shard_tokens": 100000000, + "tokenizer_threads": 8 + }, + "training": { + "device_type": "cuda", + "depth": 12, + "aspect_ratio": 64, + "head_dim": 128, + "max_seq_len": 2048, + "scaling_params": 110100912, + "target_param_data_ratio": 30.0, + "window_pattern": "L", + "device_batch_size": 16, + "total_batch_size": 524288, + "embedding_lr": 0.3, + "unembedding_lr": 0.008, + "weight_decay": 0.28, + "matrix_lr": 0.02, + "scalar_lr": 0.5, + "warmup_steps": 40, + "warmdown_ratio": 0.65, + "final_lr_frac": 0.05, + "save_every": 500, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": 500, + "core_metric_max_per_task": 50, + "sample_every": -1 + }, + "artifacts": { + "repo": "jbduran/think.nano" + }, + "wandb": { + "entity": "jbduran-thinkingmachinesncsu", + "project": "think.nano", + "name": "think-d12-1ep-65sh-r30", + "group": "think-d12-stopping-point", + "tags": [ + "think-dataset", + "d12", + "ratio30", + "65-shards", + "a100", + "bf16", + "stopping-point" + ] + }, + "config_fingerprint": "35996219a51996ca", + "artifact_path": "experiments/think-d12-1ep-65sh-r30" + }, + "stage": "base", + "base_experiment_id": "think-d12-1ep-65sh-r30", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "config_fingerprint": "35996219a51996ca" + }, + "device_batch_size": 16, + "max_seq_len": 2048, + "total_batch_size": 524288, + "dataloader_state_dict": { + "file_idx": 7, + "pos": 86488769, + "epoch": 1, + "pq_idx": 7, + "rg_idx": 86488769 + }, + "loop_state": { + "min_val_bpb": 1.2322366946944483, + "smooth_train_loss": 3.4686871369235353, + "total_training_time": 4000.771213531494, + "stage_training_flops": 697642166255616000, + "inherited_parent_flops": 0.0, + "cumulative_pipeline_training_flops": 697642166255616000 + } +} \ No newline at end of file diff --git a/experiments/think-d12-1ep-65sh-r30/base_checkpoints/meta_002000.json b/experiments/think-d12-1ep-65sh-r30/base_checkpoints/meta_002000.json new file mode 100644 index 0000000000000000000000000000000000000000..0b39b710d4315637037a45646b12a28ae3c1d613 --- /dev/null +++ b/experiments/think-d12-1ep-65sh-r30/base_checkpoints/meta_002000.json @@ -0,0 +1,155 @@ +{ + "step": 2000, + "experiment_id": "think-d12-1ep-65sh-r30", + "val_bpb": 1.212170813932778, + "model_config": { + "sequence_len": 2048, + "vocab_size": 32768, + "n_layer": 12, + "n_head": 6, + "n_kv_head": 6, + "n_embd": 768, + "window_pattern": "L" + }, + "user_config": { + "run": "think-d12-1ep-65sh-r30", + "wandb_run_id": "6465e19b", + "wandb_group": "think-d12-stopping-point", + "wandb_tags": "think-dataset,d12,ratio30,65-shards,a100,bf16,stopping-point", + "device_type": "cuda", + "fp8": false, + "fp8_recipe": "tensorwise", + "depth": 12, + "aspect_ratio": 64, + "head_dim": 128, + "max_seq_len": 2048, + "window_pattern": "L", + "num_iterations": -1, + "target_flops": -1.0, + "target_param_data_ratio": 30.0, + "device_batch_size": 16, + "total_batch_size": 524288, + "embedding_lr": 0.3, + "unembedding_lr": 0.008, + "weight_decay": 0.28, + "matrix_lr": 0.02, + "scalar_lr": 0.5, + "warmup_steps": 40, + "warmdown_ratio": 0.65, + "final_lr_frac": 0.05, + "resume_from_step": -1, + "pretokenized": true, + "data_dir": "/content/nanochat_cache/experiments/think-d12-1ep-65sh-r30/data", + "tokenizer_dir": "/content/nanochat_cache/experiments/think-d12-1ep-65sh-r30/tokenizer", + "pretokenized_dir": "/content/nanochat_cache/experiments/think-d12-1ep-65sh-r30/pretok", + "checkpoint_dir": "/content/nanochat_cache/experiments/think-d12-1ep-65sh-r30/base_checkpoints", + "experiment_id": "think-d12-1ep-65sh-r30", + "experiment_config": "/content/nanochat_cache/experiments/think-d12-1ep-65sh-r30/config.json", + "tokenizer_fingerprint": "db3bec0946e70097", + "git_commit_sha": "09b5f3b7b61a45f34d50a36e570b85f0259208bc", + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": 500, + "core_metric_max_per_task": 50, + "sample_every": -1, + "save_every": 500, + "model_tag": "think-d12-1ep-65sh-r30", + "experiment": { + "schema_version": 1, + "stage": "base", + "experiment_id": "think-d12-1ep-65sh-r30", + "dataset": { + "adapter": "parquet_shards", + "repo": "jbduran/think-dataset", + "revision": "main", + "validation_shard": 472, + "num_train_shards": 65, + "download_workers": 4 + }, + "tokenizer": { + "mode": "train", + "max_chars": 2000000000, + "doc_cap": 10000, + "vocab_size": 32768 + }, + "pretokenize": { + "enabled": true, + "target_tokens": 3402104832, + "slack": 1.03, + "require_no_wrap": true, + "val_tokens": 20971520, + "shard_tokens": 100000000, + "tokenizer_threads": 8 + }, + "training": { + "device_type": "cuda", + "depth": 12, + "aspect_ratio": 64, + "head_dim": 128, + "max_seq_len": 2048, + "scaling_params": 110100912, + "target_param_data_ratio": 30.0, + "window_pattern": "L", + "device_batch_size": 16, + "total_batch_size": 524288, + "embedding_lr": 0.3, + "unembedding_lr": 0.008, + "weight_decay": 0.28, + "matrix_lr": 0.02, + "scalar_lr": 0.5, + "warmup_steps": 40, + "warmdown_ratio": 0.65, + "final_lr_frac": 0.05, + "save_every": 500, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": 500, + "core_metric_max_per_task": 50, + "sample_every": -1 + }, + "artifacts": { + "repo": "jbduran/think.nano" + }, + "wandb": { + "entity": "jbduran-thinkingmachinesncsu", + "project": "think.nano", + "name": "think-d12-1ep-65sh-r30", + "group": "think-d12-stopping-point", + "tags": [ + "think-dataset", + "d12", + "ratio30", + "65-shards", + "a100", + "bf16", + "stopping-point" + ] + }, + "config_fingerprint": "35996219a51996ca", + "artifact_path": "experiments/think-d12-1ep-65sh-r30" + }, + "stage": "base", + "base_experiment_id": "think-d12-1ep-65sh-r30", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "config_fingerprint": "35996219a51996ca" + }, + "device_batch_size": 16, + "max_seq_len": 2048, + "total_batch_size": 524288, + "dataloader_state_dict": { + "file_idx": 10, + "pos": 48640769, + "epoch": 1, + "pq_idx": 10, + "rg_idx": 48640769 + }, + "loop_state": { + "min_val_bpb": 1.212170813932778, + "smooth_train_loss": 3.525094410637873, + "total_training_time": 5345.096604824066, + "stage_training_flops": 930189555007488000, + "inherited_parent_flops": 0.0, + "cumulative_pipeline_training_flops": 930189555007488000 + } +} \ No newline at end of file diff --git a/experiments/think-d12-1ep-65sh-r30/base_checkpoints/meta_002500.json b/experiments/think-d12-1ep-65sh-r30/base_checkpoints/meta_002500.json new file mode 100644 index 0000000000000000000000000000000000000000..11bb1070a1c10a301397fe9d81cb73880cf68005 --- /dev/null +++ b/experiments/think-d12-1ep-65sh-r30/base_checkpoints/meta_002500.json @@ -0,0 +1,155 @@ +{ + "step": 2500, + "experiment_id": "think-d12-1ep-65sh-r30", + "val_bpb": 1.1919329110762835, + "model_config": { + "sequence_len": 2048, + "vocab_size": 32768, + "n_layer": 12, + "n_head": 6, + "n_kv_head": 6, + "n_embd": 768, + "window_pattern": "L" + }, + "user_config": { + "run": "think-d12-1ep-65sh-r30", + "wandb_run_id": "6465e19b", + "wandb_group": "think-d12-stopping-point", + "wandb_tags": "think-dataset,d12,ratio30,65-shards,a100,bf16,stopping-point", + "device_type": "cuda", + "fp8": false, + "fp8_recipe": "tensorwise", + "depth": 12, + "aspect_ratio": 64, + "head_dim": 128, + "max_seq_len": 2048, + "window_pattern": "L", + "num_iterations": -1, + "target_flops": -1.0, + "target_param_data_ratio": 30.0, + "device_batch_size": 16, + "total_batch_size": 524288, + "embedding_lr": 0.3, + "unembedding_lr": 0.008, + "weight_decay": 0.28, + "matrix_lr": 0.02, + "scalar_lr": 0.5, + "warmup_steps": 40, + "warmdown_ratio": 0.65, + "final_lr_frac": 0.05, + "resume_from_step": -1, + "pretokenized": true, + "data_dir": "/content/nanochat_cache/experiments/think-d12-1ep-65sh-r30/data", + "tokenizer_dir": "/content/nanochat_cache/experiments/think-d12-1ep-65sh-r30/tokenizer", + "pretokenized_dir": "/content/nanochat_cache/experiments/think-d12-1ep-65sh-r30/pretok", + "checkpoint_dir": "/content/nanochat_cache/experiments/think-d12-1ep-65sh-r30/base_checkpoints", + "experiment_id": "think-d12-1ep-65sh-r30", + "experiment_config": "/content/nanochat_cache/experiments/think-d12-1ep-65sh-r30/config.json", + "tokenizer_fingerprint": "db3bec0946e70097", + "git_commit_sha": "09b5f3b7b61a45f34d50a36e570b85f0259208bc", + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": 500, + "core_metric_max_per_task": 50, + "sample_every": -1, + "save_every": 500, + "model_tag": "think-d12-1ep-65sh-r30", + "experiment": { + "schema_version": 1, + "stage": "base", + "experiment_id": "think-d12-1ep-65sh-r30", + "dataset": { + "adapter": "parquet_shards", + "repo": "jbduran/think-dataset", + "revision": "main", + "validation_shard": 472, + "num_train_shards": 65, + "download_workers": 4 + }, + "tokenizer": { + "mode": "train", + "max_chars": 2000000000, + "doc_cap": 10000, + "vocab_size": 32768 + }, + "pretokenize": { + "enabled": true, + "target_tokens": 3402104832, + "slack": 1.03, + "require_no_wrap": true, + "val_tokens": 20971520, + "shard_tokens": 100000000, + "tokenizer_threads": 8 + }, + "training": { + "device_type": "cuda", + "depth": 12, + "aspect_ratio": 64, + "head_dim": 128, + "max_seq_len": 2048, + "scaling_params": 110100912, + "target_param_data_ratio": 30.0, + "window_pattern": "L", + "device_batch_size": 16, + "total_batch_size": 524288, + "embedding_lr": 0.3, + "unembedding_lr": 0.008, + "weight_decay": 0.28, + "matrix_lr": 0.02, + "scalar_lr": 0.5, + "warmup_steps": 40, + "warmdown_ratio": 0.65, + "final_lr_frac": 0.05, + "save_every": 500, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": 500, + "core_metric_max_per_task": 50, + "sample_every": -1 + }, + "artifacts": { + "repo": "jbduran/think.nano" + }, + "wandb": { + "entity": "jbduran-thinkingmachinesncsu", + "project": "think.nano", + "name": "think-d12-1ep-65sh-r30", + "group": "think-d12-stopping-point", + "tags": [ + "think-dataset", + "d12", + "ratio30", + "65-shards", + "a100", + "bf16", + "stopping-point" + ] + }, + "config_fingerprint": "35996219a51996ca", + "artifact_path": "experiments/think-d12-1ep-65sh-r30" + }, + "stage": "base", + "base_experiment_id": "think-d12-1ep-65sh-r30", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "config_fingerprint": "35996219a51996ca" + }, + "device_batch_size": 16, + "max_seq_len": 2048, + "total_batch_size": 524288, + "dataloader_state_dict": { + "file_idx": 13, + "pos": 10792769, + "epoch": 1, + "pq_idx": 13, + "rg_idx": 10792769 + }, + "loop_state": { + "min_val_bpb": 1.1919329110762835, + "smooth_train_loss": 3.4785363277537242, + "total_training_time": 6687.278959035873, + "stage_training_flops": 1162736943759360000, + "inherited_parent_flops": 0.0, + "cumulative_pipeline_training_flops": 1162736943759360000 + } +} \ No newline at end of file diff --git a/experiments/think-d12-1ep-65sh-r30/base_checkpoints/meta_003000.json b/experiments/think-d12-1ep-65sh-r30/base_checkpoints/meta_003000.json new file mode 100644 index 0000000000000000000000000000000000000000..00d466b02d1cee4ab58d492c1714721d1ecd3b25 --- /dev/null +++ b/experiments/think-d12-1ep-65sh-r30/base_checkpoints/meta_003000.json @@ -0,0 +1,155 @@ +{ + "step": 3000, + "experiment_id": "think-d12-1ep-65sh-r30", + "val_bpb": 1.181723141516681, + "model_config": { + "sequence_len": 2048, + "vocab_size": 32768, + "n_layer": 12, + "n_head": 6, + "n_kv_head": 6, + "n_embd": 768, + "window_pattern": "L" + }, + "user_config": { + "run": "think-d12-1ep-65sh-r30", + "wandb_run_id": "6465e19b", + "wandb_group": "think-d12-stopping-point", + "wandb_tags": "think-dataset,d12,ratio30,65-shards,a100,bf16,stopping-point", + "device_type": "cuda", + "fp8": false, + "fp8_recipe": "tensorwise", + "depth": 12, + "aspect_ratio": 64, + "head_dim": 128, + "max_seq_len": 2048, + "window_pattern": "L", + "num_iterations": -1, + "target_flops": -1.0, + "target_param_data_ratio": 30.0, + "device_batch_size": 16, + "total_batch_size": 524288, + "embedding_lr": 0.3, + "unembedding_lr": 0.008, + "weight_decay": 0.28, + "matrix_lr": 0.02, + "scalar_lr": 0.5, + "warmup_steps": 40, + "warmdown_ratio": 0.65, + "final_lr_frac": 0.05, + "resume_from_step": -1, + "pretokenized": true, + "data_dir": "/content/nanochat_cache/experiments/think-d12-1ep-65sh-r30/data", + "tokenizer_dir": "/content/nanochat_cache/experiments/think-d12-1ep-65sh-r30/tokenizer", + "pretokenized_dir": "/content/nanochat_cache/experiments/think-d12-1ep-65sh-r30/pretok", + "checkpoint_dir": "/content/nanochat_cache/experiments/think-d12-1ep-65sh-r30/base_checkpoints", + "experiment_id": "think-d12-1ep-65sh-r30", + "experiment_config": "/content/nanochat_cache/experiments/think-d12-1ep-65sh-r30/config.json", + "tokenizer_fingerprint": "db3bec0946e70097", + "git_commit_sha": "09b5f3b7b61a45f34d50a36e570b85f0259208bc", + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": 500, + "core_metric_max_per_task": 50, + "sample_every": -1, + "save_every": 500, + "model_tag": "think-d12-1ep-65sh-r30", + "experiment": { + "schema_version": 1, + "stage": "base", + "experiment_id": "think-d12-1ep-65sh-r30", + "dataset": { + "adapter": "parquet_shards", + "repo": "jbduran/think-dataset", + "revision": "main", + "validation_shard": 472, + "num_train_shards": 65, + "download_workers": 4 + }, + "tokenizer": { + "mode": "train", + "max_chars": 2000000000, + "doc_cap": 10000, + "vocab_size": 32768 + }, + "pretokenize": { + "enabled": true, + "target_tokens": 3402104832, + "slack": 1.03, + "require_no_wrap": true, + "val_tokens": 20971520, + "shard_tokens": 100000000, + "tokenizer_threads": 8 + }, + "training": { + "device_type": "cuda", + "depth": 12, + "aspect_ratio": 64, + "head_dim": 128, + "max_seq_len": 2048, + "scaling_params": 110100912, + "target_param_data_ratio": 30.0, + "window_pattern": "L", + "device_batch_size": 16, + "total_batch_size": 524288, + "embedding_lr": 0.3, + "unembedding_lr": 0.008, + "weight_decay": 0.28, + "matrix_lr": 0.02, + "scalar_lr": 0.5, + "warmup_steps": 40, + "warmdown_ratio": 0.65, + "final_lr_frac": 0.05, + "save_every": 500, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": 500, + "core_metric_max_per_task": 50, + "sample_every": -1 + }, + "artifacts": { + "repo": "jbduran/think.nano" + }, + "wandb": { + "entity": "jbduran-thinkingmachinesncsu", + "project": "think.nano", + "name": "think-d12-1ep-65sh-r30", + "group": "think-d12-stopping-point", + "tags": [ + "think-dataset", + "d12", + "ratio30", + "65-shards", + "a100", + "bf16", + "stopping-point" + ] + }, + "config_fingerprint": "35996219a51996ca", + "artifact_path": "experiments/think-d12-1ep-65sh-r30" + }, + "stage": "base", + "base_experiment_id": "think-d12-1ep-65sh-r30", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "config_fingerprint": "35996219a51996ca" + }, + "device_batch_size": 16, + "max_seq_len": 2048, + "total_batch_size": 524288, + "dataloader_state_dict": { + "file_idx": 15, + "pos": 72944769, + "epoch": 1, + "pq_idx": 15, + "rg_idx": 72944769 + }, + "loop_state": { + "min_val_bpb": 1.181723141516681, + "smooth_train_loss": 3.1625074370797437, + "total_training_time": 8029.638848543167, + "stage_training_flops": 1395284332511232000, + "inherited_parent_flops": 0.0, + "cumulative_pipeline_training_flops": 1395284332511232000 + } +} \ No newline at end of file diff --git a/experiments/think-d12-1ep-65sh-r30/base_checkpoints/meta_003500.json b/experiments/think-d12-1ep-65sh-r30/base_checkpoints/meta_003500.json new file mode 100644 index 0000000000000000000000000000000000000000..ae6949445152310ef99e38c07adf90d8979be3d8 --- /dev/null +++ b/experiments/think-d12-1ep-65sh-r30/base_checkpoints/meta_003500.json @@ -0,0 +1,155 @@ +{ + "step": 3500, + "experiment_id": "think-d12-1ep-65sh-r30", + "val_bpb": 1.1594974101297872, + "model_config": { + "sequence_len": 2048, + "vocab_size": 32768, + "n_layer": 12, + "n_head": 6, + "n_kv_head": 6, + "n_embd": 768, + "window_pattern": "L" + }, + "user_config": { + "run": "think-d12-1ep-65sh-r30", + "wandb_run_id": "6465e19b", + "wandb_group": "think-d12-stopping-point", + "wandb_tags": "think-dataset,d12,ratio30,65-shards,a100,bf16,stopping-point", + "device_type": "cuda", + "fp8": false, + "fp8_recipe": "tensorwise", + "depth": 12, + "aspect_ratio": 64, + "head_dim": 128, + "max_seq_len": 2048, + "window_pattern": "L", + "num_iterations": -1, + "target_flops": -1.0, + "target_param_data_ratio": 30.0, + "device_batch_size": 16, + "total_batch_size": 524288, + "embedding_lr": 0.3, + "unembedding_lr": 0.008, + "weight_decay": 0.28, + "matrix_lr": 0.02, + "scalar_lr": 0.5, + "warmup_steps": 40, + "warmdown_ratio": 0.65, + "final_lr_frac": 0.05, + "resume_from_step": -1, + "pretokenized": true, + "data_dir": "/content/nanochat_cache/experiments/think-d12-1ep-65sh-r30/data", + "tokenizer_dir": "/content/nanochat_cache/experiments/think-d12-1ep-65sh-r30/tokenizer", + "pretokenized_dir": "/content/nanochat_cache/experiments/think-d12-1ep-65sh-r30/pretok", + "checkpoint_dir": "/content/nanochat_cache/experiments/think-d12-1ep-65sh-r30/base_checkpoints", + "experiment_id": "think-d12-1ep-65sh-r30", + "experiment_config": "/content/nanochat_cache/experiments/think-d12-1ep-65sh-r30/config.json", + "tokenizer_fingerprint": "db3bec0946e70097", + "git_commit_sha": "09b5f3b7b61a45f34d50a36e570b85f0259208bc", + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": 500, + "core_metric_max_per_task": 50, + "sample_every": -1, + "save_every": 500, + "model_tag": "think-d12-1ep-65sh-r30", + "experiment": { + "schema_version": 1, + "stage": "base", + "experiment_id": "think-d12-1ep-65sh-r30", + "dataset": { + "adapter": "parquet_shards", + "repo": "jbduran/think-dataset", + "revision": "main", + "validation_shard": 472, + "num_train_shards": 65, + "download_workers": 4 + }, + "tokenizer": { + "mode": "train", + "max_chars": 2000000000, + "doc_cap": 10000, + "vocab_size": 32768 + }, + "pretokenize": { + "enabled": true, + "target_tokens": 3402104832, + "slack": 1.03, + "require_no_wrap": true, + "val_tokens": 20971520, + "shard_tokens": 100000000, + "tokenizer_threads": 8 + }, + "training": { + "device_type": "cuda", + "depth": 12, + "aspect_ratio": 64, + "head_dim": 128, + "max_seq_len": 2048, + "scaling_params": 110100912, + "target_param_data_ratio": 30.0, + "window_pattern": "L", + "device_batch_size": 16, + "total_batch_size": 524288, + "embedding_lr": 0.3, + "unembedding_lr": 0.008, + "weight_decay": 0.28, + "matrix_lr": 0.02, + "scalar_lr": 0.5, + "warmup_steps": 40, + "warmdown_ratio": 0.65, + "final_lr_frac": 0.05, + "save_every": 500, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": 500, + "core_metric_max_per_task": 50, + "sample_every": -1 + }, + "artifacts": { + "repo": "jbduran/think.nano" + }, + "wandb": { + "entity": "jbduran-thinkingmachinesncsu", + "project": "think.nano", + "name": "think-d12-1ep-65sh-r30", + "group": "think-d12-stopping-point", + "tags": [ + "think-dataset", + "d12", + "ratio30", + "65-shards", + "a100", + "bf16", + "stopping-point" + ] + }, + "config_fingerprint": "35996219a51996ca", + "artifact_path": "experiments/think-d12-1ep-65sh-r30" + }, + "stage": "base", + "base_experiment_id": "think-d12-1ep-65sh-r30", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "config_fingerprint": "35996219a51996ca" + }, + "device_batch_size": 16, + "max_seq_len": 2048, + "total_batch_size": 524288, + "dataloader_state_dict": { + "file_idx": 18, + "pos": 35096769, + "epoch": 1, + "pq_idx": 18, + "rg_idx": 35096769 + }, + "loop_state": { + "min_val_bpb": 1.1594974101297872, + "smooth_train_loss": 3.1518777563692284, + "total_training_time": 9373.077644109726, + "stage_training_flops": 1627831721263104000, + "inherited_parent_flops": 0.0, + "cumulative_pipeline_training_flops": 1627831721263104000 + } +} \ No newline at end of file diff --git a/experiments/think-d12-1ep-65sh-r30/base_checkpoints/meta_004000.json b/experiments/think-d12-1ep-65sh-r30/base_checkpoints/meta_004000.json new file mode 100644 index 0000000000000000000000000000000000000000..0aa1925d4fe0404ed0e23cd19843fe2ea034659f --- /dev/null +++ b/experiments/think-d12-1ep-65sh-r30/base_checkpoints/meta_004000.json @@ -0,0 +1,155 @@ +{ + "step": 4000, + "experiment_id": "think-d12-1ep-65sh-r30", + "val_bpb": 1.1364783907083784, + "model_config": { + "sequence_len": 2048, + "vocab_size": 32768, + "n_layer": 12, + "n_head": 6, + "n_kv_head": 6, + "n_embd": 768, + "window_pattern": "L" + }, + "user_config": { + "run": "think-d12-1ep-65sh-r30", + "wandb_run_id": "6465e19b", + "wandb_group": "think-d12-stopping-point", + "wandb_tags": "think-dataset,d12,ratio30,65-shards,a100,bf16,stopping-point", + "device_type": "cuda", + "fp8": false, + "fp8_recipe": "tensorwise", + "depth": 12, + "aspect_ratio": 64, + "head_dim": 128, + "max_seq_len": 2048, + "window_pattern": "L", + "num_iterations": -1, + "target_flops": -1.0, + "target_param_data_ratio": 30.0, + "device_batch_size": 16, + "total_batch_size": 524288, + "embedding_lr": 0.3, + "unembedding_lr": 0.008, + "weight_decay": 0.28, + "matrix_lr": 0.02, + "scalar_lr": 0.5, + "warmup_steps": 40, + "warmdown_ratio": 0.65, + "final_lr_frac": 0.05, + "resume_from_step": -1, + "pretokenized": true, + "data_dir": "/content/nanochat_cache/experiments/think-d12-1ep-65sh-r30/data", + "tokenizer_dir": "/content/nanochat_cache/experiments/think-d12-1ep-65sh-r30/tokenizer", + "pretokenized_dir": "/content/nanochat_cache/experiments/think-d12-1ep-65sh-r30/pretok", + "checkpoint_dir": "/content/nanochat_cache/experiments/think-d12-1ep-65sh-r30/base_checkpoints", + "experiment_id": "think-d12-1ep-65sh-r30", + "experiment_config": "/content/nanochat_cache/experiments/think-d12-1ep-65sh-r30/config.json", + "tokenizer_fingerprint": "db3bec0946e70097", + "git_commit_sha": "09b5f3b7b61a45f34d50a36e570b85f0259208bc", + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": 500, + "core_metric_max_per_task": 50, + "sample_every": -1, + "save_every": 500, + "model_tag": "think-d12-1ep-65sh-r30", + "experiment": { + "schema_version": 1, + "stage": "base", + "experiment_id": "think-d12-1ep-65sh-r30", + "dataset": { + "adapter": "parquet_shards", + "repo": "jbduran/think-dataset", + "revision": "main", + "validation_shard": 472, + "num_train_shards": 65, + "download_workers": 4 + }, + "tokenizer": { + "mode": "train", + "max_chars": 2000000000, + "doc_cap": 10000, + "vocab_size": 32768 + }, + "pretokenize": { + "enabled": true, + "target_tokens": 3402104832, + "slack": 1.03, + "require_no_wrap": true, + "val_tokens": 20971520, + "shard_tokens": 100000000, + "tokenizer_threads": 8 + }, + "training": { + "device_type": "cuda", + "depth": 12, + "aspect_ratio": 64, + "head_dim": 128, + "max_seq_len": 2048, + "scaling_params": 110100912, + "target_param_data_ratio": 30.0, + "window_pattern": "L", + "device_batch_size": 16, + "total_batch_size": 524288, + "embedding_lr": 0.3, + "unembedding_lr": 0.008, + "weight_decay": 0.28, + "matrix_lr": 0.02, + "scalar_lr": 0.5, + "warmup_steps": 40, + "warmdown_ratio": 0.65, + "final_lr_frac": 0.05, + "save_every": 500, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": 500, + "core_metric_max_per_task": 50, + "sample_every": -1 + }, + "artifacts": { + "repo": "jbduran/think.nano" + }, + "wandb": { + "entity": "jbduran-thinkingmachinesncsu", + "project": "think.nano", + "name": "think-d12-1ep-65sh-r30", + "group": "think-d12-stopping-point", + "tags": [ + "think-dataset", + "d12", + "ratio30", + "65-shards", + "a100", + "bf16", + "stopping-point" + ] + }, + "config_fingerprint": "35996219a51996ca", + "artifact_path": "experiments/think-d12-1ep-65sh-r30" + }, + "stage": "base", + "base_experiment_id": "think-d12-1ep-65sh-r30", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "config_fingerprint": "35996219a51996ca" + }, + "device_batch_size": 16, + "max_seq_len": 2048, + "total_batch_size": 524288, + "dataloader_state_dict": { + "file_idx": 20, + "pos": 97248769, + "epoch": 1, + "pq_idx": 20, + "rg_idx": 97248769 + }, + "loop_state": { + "min_val_bpb": 1.1364783907083784, + "smooth_train_loss": 3.090057974445858, + "total_training_time": 10716.081592082977, + "stage_training_flops": 1860379110014976000, + "inherited_parent_flops": 0.0, + "cumulative_pipeline_training_flops": 1860379110014976000 + } +} \ No newline at end of file diff --git a/experiments/think-d12-1ep-65sh-r30/base_checkpoints/meta_004500.json b/experiments/think-d12-1ep-65sh-r30/base_checkpoints/meta_004500.json new file mode 100644 index 0000000000000000000000000000000000000000..a18bc3fc05e6e76f00d53bed2fd391962955762d --- /dev/null +++ b/experiments/think-d12-1ep-65sh-r30/base_checkpoints/meta_004500.json @@ -0,0 +1,155 @@ +{ + "step": 4500, + "experiment_id": "think-d12-1ep-65sh-r30", + "val_bpb": 1.1270159041981165, + "model_config": { + "sequence_len": 2048, + "vocab_size": 32768, + "n_layer": 12, + "n_head": 6, + "n_kv_head": 6, + "n_embd": 768, + "window_pattern": "L" + }, + "user_config": { + "run": "think-d12-1ep-65sh-r30", + "wandb_run_id": "6465e19b", + "wandb_group": "think-d12-stopping-point", + "wandb_tags": "think-dataset,d12,ratio30,65-shards,a100,bf16,stopping-point", + "device_type": "cuda", + "fp8": false, + "fp8_recipe": "tensorwise", + "depth": 12, + "aspect_ratio": 64, + "head_dim": 128, + "max_seq_len": 2048, + "window_pattern": "L", + "num_iterations": -1, + "target_flops": -1.0, + "target_param_data_ratio": 30.0, + "device_batch_size": 16, + "total_batch_size": 524288, + "embedding_lr": 0.3, + "unembedding_lr": 0.008, + "weight_decay": 0.28, + "matrix_lr": 0.02, + "scalar_lr": 0.5, + "warmup_steps": 40, + "warmdown_ratio": 0.65, + "final_lr_frac": 0.05, + "resume_from_step": -1, + "pretokenized": true, + "data_dir": "/content/nanochat_cache/experiments/think-d12-1ep-65sh-r30/data", + "tokenizer_dir": "/content/nanochat_cache/experiments/think-d12-1ep-65sh-r30/tokenizer", + "pretokenized_dir": "/content/nanochat_cache/experiments/think-d12-1ep-65sh-r30/pretok", + "checkpoint_dir": "/content/nanochat_cache/experiments/think-d12-1ep-65sh-r30/base_checkpoints", + "experiment_id": "think-d12-1ep-65sh-r30", + "experiment_config": "/content/nanochat_cache/experiments/think-d12-1ep-65sh-r30/config.json", + "tokenizer_fingerprint": "db3bec0946e70097", + "git_commit_sha": "09b5f3b7b61a45f34d50a36e570b85f0259208bc", + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": 500, + "core_metric_max_per_task": 50, + "sample_every": -1, + "save_every": 500, + "model_tag": "think-d12-1ep-65sh-r30", + "experiment": { + "schema_version": 1, + "stage": "base", + "experiment_id": "think-d12-1ep-65sh-r30", + "dataset": { + "adapter": "parquet_shards", + "repo": "jbduran/think-dataset", + "revision": "main", + "validation_shard": 472, + "num_train_shards": 65, + "download_workers": 4 + }, + "tokenizer": { + "mode": "train", + "max_chars": 2000000000, + "doc_cap": 10000, + "vocab_size": 32768 + }, + "pretokenize": { + "enabled": true, + "target_tokens": 3402104832, + "slack": 1.03, + "require_no_wrap": true, + "val_tokens": 20971520, + "shard_tokens": 100000000, + "tokenizer_threads": 8 + }, + "training": { + "device_type": "cuda", + "depth": 12, + "aspect_ratio": 64, + "head_dim": 128, + "max_seq_len": 2048, + "scaling_params": 110100912, + "target_param_data_ratio": 30.0, + "window_pattern": "L", + "device_batch_size": 16, + "total_batch_size": 524288, + "embedding_lr": 0.3, + "unembedding_lr": 0.008, + "weight_decay": 0.28, + "matrix_lr": 0.02, + "scalar_lr": 0.5, + "warmup_steps": 40, + "warmdown_ratio": 0.65, + "final_lr_frac": 0.05, + "save_every": 500, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": 500, + "core_metric_max_per_task": 50, + "sample_every": -1 + }, + "artifacts": { + "repo": "jbduran/think.nano" + }, + "wandb": { + "entity": "jbduran-thinkingmachinesncsu", + "project": "think.nano", + "name": "think-d12-1ep-65sh-r30", + "group": "think-d12-stopping-point", + "tags": [ + "think-dataset", + "d12", + "ratio30", + "65-shards", + "a100", + "bf16", + "stopping-point" + ] + }, + "config_fingerprint": "35996219a51996ca", + "artifact_path": "experiments/think-d12-1ep-65sh-r30" + }, + "stage": "base", + "base_experiment_id": "think-d12-1ep-65sh-r30", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "config_fingerprint": "35996219a51996ca" + }, + "device_batch_size": 16, + "max_seq_len": 2048, + "total_batch_size": 524288, + "dataloader_state_dict": { + "file_idx": 23, + "pos": 59400769, + "epoch": 1, + "pq_idx": 23, + "rg_idx": 59400769 + }, + "loop_state": { + "min_val_bpb": 1.1270159041981165, + "smooth_train_loss": 2.9730753725244097, + "total_training_time": 12057.107451677322, + "stage_training_flops": 2092926498766848000, + "inherited_parent_flops": 0.0, + "cumulative_pipeline_training_flops": 2092926498766848000 + } +} \ No newline at end of file diff --git a/experiments/think-d12-1ep-65sh-r30/base_checkpoints/meta_005000.json b/experiments/think-d12-1ep-65sh-r30/base_checkpoints/meta_005000.json new file mode 100644 index 0000000000000000000000000000000000000000..e4b118dbbc6d587759704a6ae9feaeb05bb343ab --- /dev/null +++ b/experiments/think-d12-1ep-65sh-r30/base_checkpoints/meta_005000.json @@ -0,0 +1,155 @@ +{ + "step": 5000, + "experiment_id": "think-d12-1ep-65sh-r30", + "val_bpb": 1.1003152146678035, + "model_config": { + "sequence_len": 2048, + "vocab_size": 32768, + "n_layer": 12, + "n_head": 6, + "n_kv_head": 6, + "n_embd": 768, + "window_pattern": "L" + }, + "user_config": { + "run": "think-d12-1ep-65sh-r30", + "wandb_run_id": "6465e19b", + "wandb_group": "think-d12-stopping-point", + "wandb_tags": "think-dataset,d12,ratio30,65-shards,a100,bf16,stopping-point", + "device_type": "cuda", + "fp8": false, + "fp8_recipe": "tensorwise", + "depth": 12, + "aspect_ratio": 64, + "head_dim": 128, + "max_seq_len": 2048, + "window_pattern": "L", + "num_iterations": -1, + "target_flops": -1.0, + "target_param_data_ratio": 30.0, + "device_batch_size": 16, + "total_batch_size": 524288, + "embedding_lr": 0.3, + "unembedding_lr": 0.008, + "weight_decay": 0.28, + "matrix_lr": 0.02, + "scalar_lr": 0.5, + "warmup_steps": 40, + "warmdown_ratio": 0.65, + "final_lr_frac": 0.05, + "resume_from_step": -1, + "pretokenized": true, + "data_dir": "/content/nanochat_cache/experiments/think-d12-1ep-65sh-r30/data", + "tokenizer_dir": "/content/nanochat_cache/experiments/think-d12-1ep-65sh-r30/tokenizer", + "pretokenized_dir": "/content/nanochat_cache/experiments/think-d12-1ep-65sh-r30/pretok", + "checkpoint_dir": "/content/nanochat_cache/experiments/think-d12-1ep-65sh-r30/base_checkpoints", + "experiment_id": "think-d12-1ep-65sh-r30", + "experiment_config": "/content/nanochat_cache/experiments/think-d12-1ep-65sh-r30/config.json", + "tokenizer_fingerprint": "db3bec0946e70097", + "git_commit_sha": "09b5f3b7b61a45f34d50a36e570b85f0259208bc", + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": 500, + "core_metric_max_per_task": 50, + "sample_every": -1, + "save_every": 500, + "model_tag": "think-d12-1ep-65sh-r30", + "experiment": { + "schema_version": 1, + "stage": "base", + "experiment_id": "think-d12-1ep-65sh-r30", + "dataset": { + "adapter": "parquet_shards", + "repo": "jbduran/think-dataset", + "revision": "main", + "validation_shard": 472, + "num_train_shards": 65, + "download_workers": 4 + }, + "tokenizer": { + "mode": "train", + "max_chars": 2000000000, + "doc_cap": 10000, + "vocab_size": 32768 + }, + "pretokenize": { + "enabled": true, + "target_tokens": 3402104832, + "slack": 1.03, + "require_no_wrap": true, + "val_tokens": 20971520, + "shard_tokens": 100000000, + "tokenizer_threads": 8 + }, + "training": { + "device_type": "cuda", + "depth": 12, + "aspect_ratio": 64, + "head_dim": 128, + "max_seq_len": 2048, + "scaling_params": 110100912, + "target_param_data_ratio": 30.0, + "window_pattern": "L", + "device_batch_size": 16, + "total_batch_size": 524288, + "embedding_lr": 0.3, + "unembedding_lr": 0.008, + "weight_decay": 0.28, + "matrix_lr": 0.02, + "scalar_lr": 0.5, + "warmup_steps": 40, + "warmdown_ratio": 0.65, + "final_lr_frac": 0.05, + "save_every": 500, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": 500, + "core_metric_max_per_task": 50, + "sample_every": -1 + }, + "artifacts": { + "repo": "jbduran/think.nano" + }, + "wandb": { + "entity": "jbduran-thinkingmachinesncsu", + "project": "think.nano", + "name": "think-d12-1ep-65sh-r30", + "group": "think-d12-stopping-point", + "tags": [ + "think-dataset", + "d12", + "ratio30", + "65-shards", + "a100", + "bf16", + "stopping-point" + ] + }, + "config_fingerprint": "35996219a51996ca", + "artifact_path": "experiments/think-d12-1ep-65sh-r30" + }, + "stage": "base", + "base_experiment_id": "think-d12-1ep-65sh-r30", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "config_fingerprint": "35996219a51996ca" + }, + "device_batch_size": 16, + "max_seq_len": 2048, + "total_batch_size": 524288, + "dataloader_state_dict": { + "file_idx": 26, + "pos": 21552769, + "epoch": 1, + "pq_idx": 26, + "rg_idx": 21552769 + }, + "loop_state": { + "min_val_bpb": 1.1003152146678035, + "smooth_train_loss": 2.9552833417519717, + "total_training_time": 13398.46120429039, + "stage_training_flops": 2325473887518720000, + "inherited_parent_flops": 0.0, + "cumulative_pipeline_training_flops": 2325473887518720000 + } +} \ No newline at end of file diff --git a/experiments/think-d12-1ep-65sh-r30/base_checkpoints/meta_005500.json b/experiments/think-d12-1ep-65sh-r30/base_checkpoints/meta_005500.json new file mode 100644 index 0000000000000000000000000000000000000000..f23a21159bd966c66f10b6168c615c98559659b5 --- /dev/null +++ b/experiments/think-d12-1ep-65sh-r30/base_checkpoints/meta_005500.json @@ -0,0 +1,155 @@ +{ + "step": 5500, + "experiment_id": "think-d12-1ep-65sh-r30", + "val_bpb": 1.0836933513196012, + "model_config": { + "sequence_len": 2048, + "vocab_size": 32768, + "n_layer": 12, + "n_head": 6, + "n_kv_head": 6, + "n_embd": 768, + "window_pattern": "L" + }, + "user_config": { + "run": "think-d12-1ep-65sh-r30", + "wandb_run_id": "6465e19b", + "wandb_group": "think-d12-stopping-point", + "wandb_tags": "think-dataset,d12,ratio30,65-shards,a100,bf16,stopping-point", + "device_type": "cuda", + "fp8": false, + "fp8_recipe": "tensorwise", + "depth": 12, + "aspect_ratio": 64, + "head_dim": 128, + "max_seq_len": 2048, + "window_pattern": "L", + "num_iterations": -1, + "target_flops": -1.0, + "target_param_data_ratio": 30.0, + "device_batch_size": 16, + "total_batch_size": 524288, + "embedding_lr": 0.3, + "unembedding_lr": 0.008, + "weight_decay": 0.28, + "matrix_lr": 0.02, + "scalar_lr": 0.5, + "warmup_steps": 40, + "warmdown_ratio": 0.65, + "final_lr_frac": 0.05, + "resume_from_step": -1, + "pretokenized": true, + "data_dir": "/content/nanochat_cache/experiments/think-d12-1ep-65sh-r30/data", + "tokenizer_dir": "/content/nanochat_cache/experiments/think-d12-1ep-65sh-r30/tokenizer", + "pretokenized_dir": "/content/nanochat_cache/experiments/think-d12-1ep-65sh-r30/pretok", + "checkpoint_dir": "/content/nanochat_cache/experiments/think-d12-1ep-65sh-r30/base_checkpoints", + "experiment_id": "think-d12-1ep-65sh-r30", + "experiment_config": "/content/nanochat_cache/experiments/think-d12-1ep-65sh-r30/config.json", + "tokenizer_fingerprint": "db3bec0946e70097", + "git_commit_sha": "09b5f3b7b61a45f34d50a36e570b85f0259208bc", + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": 500, + "core_metric_max_per_task": 50, + "sample_every": -1, + "save_every": 500, + "model_tag": "think-d12-1ep-65sh-r30", + "experiment": { + "schema_version": 1, + "stage": "base", + "experiment_id": "think-d12-1ep-65sh-r30", + "dataset": { + "adapter": "parquet_shards", + "repo": "jbduran/think-dataset", + "revision": "main", + "validation_shard": 472, + "num_train_shards": 65, + "download_workers": 4 + }, + "tokenizer": { + "mode": "train", + "max_chars": 2000000000, + "doc_cap": 10000, + "vocab_size": 32768 + }, + "pretokenize": { + "enabled": true, + "target_tokens": 3402104832, + "slack": 1.03, + "require_no_wrap": true, + "val_tokens": 20971520, + "shard_tokens": 100000000, + "tokenizer_threads": 8 + }, + "training": { + "device_type": "cuda", + "depth": 12, + "aspect_ratio": 64, + "head_dim": 128, + "max_seq_len": 2048, + "scaling_params": 110100912, + "target_param_data_ratio": 30.0, + "window_pattern": "L", + "device_batch_size": 16, + "total_batch_size": 524288, + "embedding_lr": 0.3, + "unembedding_lr": 0.008, + "weight_decay": 0.28, + "matrix_lr": 0.02, + "scalar_lr": 0.5, + "warmup_steps": 40, + "warmdown_ratio": 0.65, + "final_lr_frac": 0.05, + "save_every": 500, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": 500, + "core_metric_max_per_task": 50, + "sample_every": -1 + }, + "artifacts": { + "repo": "jbduran/think.nano" + }, + "wandb": { + "entity": "jbduran-thinkingmachinesncsu", + "project": "think.nano", + "name": "think-d12-1ep-65sh-r30", + "group": "think-d12-stopping-point", + "tags": [ + "think-dataset", + "d12", + "ratio30", + "65-shards", + "a100", + "bf16", + "stopping-point" + ] + }, + "config_fingerprint": "35996219a51996ca", + "artifact_path": "experiments/think-d12-1ep-65sh-r30" + }, + "stage": "base", + "base_experiment_id": "think-d12-1ep-65sh-r30", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "config_fingerprint": "35996219a51996ca" + }, + "device_batch_size": 16, + "max_seq_len": 2048, + "total_batch_size": 524288, + "dataloader_state_dict": { + "file_idx": 28, + "pos": 83704769, + "epoch": 1, + "pq_idx": 28, + "rg_idx": 83704769 + }, + "loop_state": { + "min_val_bpb": 1.0836933513196012, + "smooth_train_loss": 2.8071836344710714, + "total_training_time": 14740.56656551361, + "stage_training_flops": 2558021276270592000, + "inherited_parent_flops": 0.0, + "cumulative_pipeline_training_flops": 2558021276270592000 + } +} \ No newline at end of file diff --git a/experiments/think-d12-1ep-65sh-r30/base_checkpoints/meta_006000.json b/experiments/think-d12-1ep-65sh-r30/base_checkpoints/meta_006000.json new file mode 100644 index 0000000000000000000000000000000000000000..7f68b335bb6a082f9bd1b26bcfb5e66a345861ea --- /dev/null +++ b/experiments/think-d12-1ep-65sh-r30/base_checkpoints/meta_006000.json @@ -0,0 +1,155 @@ +{ + "step": 6000, + "experiment_id": "think-d12-1ep-65sh-r30", + "val_bpb": 1.066233148422904, + "model_config": { + "sequence_len": 2048, + "vocab_size": 32768, + "n_layer": 12, + "n_head": 6, + "n_kv_head": 6, + "n_embd": 768, + "window_pattern": "L" + }, + "user_config": { + "run": "think-d12-1ep-65sh-r30", + "wandb_run_id": "6465e19b", + "wandb_group": "think-d12-stopping-point", + "wandb_tags": "think-dataset,d12,ratio30,65-shards,a100,bf16,stopping-point", + "device_type": "cuda", + "fp8": false, + "fp8_recipe": "tensorwise", + "depth": 12, + "aspect_ratio": 64, + "head_dim": 128, + "max_seq_len": 2048, + "window_pattern": "L", + "num_iterations": -1, + "target_flops": -1.0, + "target_param_data_ratio": 30.0, + "device_batch_size": 16, + "total_batch_size": 524288, + "embedding_lr": 0.3, + "unembedding_lr": 0.008, + "weight_decay": 0.28, + "matrix_lr": 0.02, + "scalar_lr": 0.5, + "warmup_steps": 40, + "warmdown_ratio": 0.65, + "final_lr_frac": 0.05, + "resume_from_step": -1, + "pretokenized": true, + "data_dir": "/content/nanochat_cache/experiments/think-d12-1ep-65sh-r30/data", + "tokenizer_dir": "/content/nanochat_cache/experiments/think-d12-1ep-65sh-r30/tokenizer", + "pretokenized_dir": "/content/nanochat_cache/experiments/think-d12-1ep-65sh-r30/pretok", + "checkpoint_dir": "/content/nanochat_cache/experiments/think-d12-1ep-65sh-r30/base_checkpoints", + "experiment_id": "think-d12-1ep-65sh-r30", + "experiment_config": "/content/nanochat_cache/experiments/think-d12-1ep-65sh-r30/config.json", + "tokenizer_fingerprint": "db3bec0946e70097", + "git_commit_sha": "09b5f3b7b61a45f34d50a36e570b85f0259208bc", + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": 500, + "core_metric_max_per_task": 50, + "sample_every": -1, + "save_every": 500, + "model_tag": "think-d12-1ep-65sh-r30", + "experiment": { + "schema_version": 1, + "stage": "base", + "experiment_id": "think-d12-1ep-65sh-r30", + "dataset": { + "adapter": "parquet_shards", + "repo": "jbduran/think-dataset", + "revision": "main", + "validation_shard": 472, + "num_train_shards": 65, + "download_workers": 4 + }, + "tokenizer": { + "mode": "train", + "max_chars": 2000000000, + "doc_cap": 10000, + "vocab_size": 32768 + }, + "pretokenize": { + "enabled": true, + "target_tokens": 3402104832, + "slack": 1.03, + "require_no_wrap": true, + "val_tokens": 20971520, + "shard_tokens": 100000000, + "tokenizer_threads": 8 + }, + "training": { + "device_type": "cuda", + "depth": 12, + "aspect_ratio": 64, + "head_dim": 128, + "max_seq_len": 2048, + "scaling_params": 110100912, + "target_param_data_ratio": 30.0, + "window_pattern": "L", + "device_batch_size": 16, + "total_batch_size": 524288, + "embedding_lr": 0.3, + "unembedding_lr": 0.008, + "weight_decay": 0.28, + "matrix_lr": 0.02, + "scalar_lr": 0.5, + "warmup_steps": 40, + "warmdown_ratio": 0.65, + "final_lr_frac": 0.05, + "save_every": 500, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": 500, + "core_metric_max_per_task": 50, + "sample_every": -1 + }, + "artifacts": { + "repo": "jbduran/think.nano" + }, + "wandb": { + "entity": "jbduran-thinkingmachinesncsu", + "project": "think.nano", + "name": "think-d12-1ep-65sh-r30", + "group": "think-d12-stopping-point", + "tags": [ + "think-dataset", + "d12", + "ratio30", + "65-shards", + "a100", + "bf16", + "stopping-point" + ] + }, + "config_fingerprint": "35996219a51996ca", + "artifact_path": "experiments/think-d12-1ep-65sh-r30" + }, + "stage": "base", + "base_experiment_id": "think-d12-1ep-65sh-r30", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "config_fingerprint": "35996219a51996ca" + }, + "device_batch_size": 16, + "max_seq_len": 2048, + "total_batch_size": 524288, + "dataloader_state_dict": { + "file_idx": 31, + "pos": 45856769, + "epoch": 1, + "pq_idx": 31, + "rg_idx": 45856769 + }, + "loop_state": { + "min_val_bpb": 1.066233148422904, + "smooth_train_loss": 2.9179841120344037, + "total_training_time": 16083.970601320267, + "stage_training_flops": 2790568665022464000, + "inherited_parent_flops": 0.0, + "cumulative_pipeline_training_flops": 2790568665022464000 + } +} \ No newline at end of file diff --git a/experiments/think-d12-1ep-65sh-r30/base_checkpoints/meta_006300.json b/experiments/think-d12-1ep-65sh-r30/base_checkpoints/meta_006300.json new file mode 100644 index 0000000000000000000000000000000000000000..e5b295c12cf7ffcff903cda2870d430aad3f06b1 --- /dev/null +++ b/experiments/think-d12-1ep-65sh-r30/base_checkpoints/meta_006300.json @@ -0,0 +1,155 @@ +{ + "step": 6300, + "experiment_id": "think-d12-1ep-65sh-r30", + "val_bpb": 1.0599363451930062, + "model_config": { + "sequence_len": 2048, + "vocab_size": 32768, + "n_layer": 12, + "n_head": 6, + "n_kv_head": 6, + "n_embd": 768, + "window_pattern": "L" + }, + "user_config": { + "run": "think-d12-1ep-65sh-r30", + "wandb_run_id": "6465e19b", + "wandb_group": "think-d12-stopping-point", + "wandb_tags": "think-dataset,d12,ratio30,65-shards,a100,bf16,stopping-point", + "device_type": "cuda", + "fp8": false, + "fp8_recipe": "tensorwise", + "depth": 12, + "aspect_ratio": 64, + "head_dim": 128, + "max_seq_len": 2048, + "window_pattern": "L", + "num_iterations": -1, + "target_flops": -1.0, + "target_param_data_ratio": 30.0, + "device_batch_size": 16, + "total_batch_size": 524288, + "embedding_lr": 0.3, + "unembedding_lr": 0.008, + "weight_decay": 0.28, + "matrix_lr": 0.02, + "scalar_lr": 0.5, + "warmup_steps": 40, + "warmdown_ratio": 0.65, + "final_lr_frac": 0.05, + "resume_from_step": -1, + "pretokenized": true, + "data_dir": "/content/nanochat_cache/experiments/think-d12-1ep-65sh-r30/data", + "tokenizer_dir": "/content/nanochat_cache/experiments/think-d12-1ep-65sh-r30/tokenizer", + "pretokenized_dir": "/content/nanochat_cache/experiments/think-d12-1ep-65sh-r30/pretok", + "checkpoint_dir": "/content/nanochat_cache/experiments/think-d12-1ep-65sh-r30/base_checkpoints", + "experiment_id": "think-d12-1ep-65sh-r30", + "experiment_config": "/content/nanochat_cache/experiments/think-d12-1ep-65sh-r30/config.json", + "tokenizer_fingerprint": "db3bec0946e70097", + "git_commit_sha": "09b5f3b7b61a45f34d50a36e570b85f0259208bc", + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": 500, + "core_metric_max_per_task": 50, + "sample_every": -1, + "save_every": 500, + "model_tag": "think-d12-1ep-65sh-r30", + "experiment": { + "schema_version": 1, + "stage": "base", + "experiment_id": "think-d12-1ep-65sh-r30", + "dataset": { + "adapter": "parquet_shards", + "repo": "jbduran/think-dataset", + "revision": "main", + "validation_shard": 472, + "num_train_shards": 65, + "download_workers": 4 + }, + "tokenizer": { + "mode": "train", + "max_chars": 2000000000, + "doc_cap": 10000, + "vocab_size": 32768 + }, + "pretokenize": { + "enabled": true, + "target_tokens": 3402104832, + "slack": 1.03, + "require_no_wrap": true, + "val_tokens": 20971520, + "shard_tokens": 100000000, + "tokenizer_threads": 8 + }, + "training": { + "device_type": "cuda", + "depth": 12, + "aspect_ratio": 64, + "head_dim": 128, + "max_seq_len": 2048, + "scaling_params": 110100912, + "target_param_data_ratio": 30.0, + "window_pattern": "L", + "device_batch_size": 16, + "total_batch_size": 524288, + "embedding_lr": 0.3, + "unembedding_lr": 0.008, + "weight_decay": 0.28, + "matrix_lr": 0.02, + "scalar_lr": 0.5, + "warmup_steps": 40, + "warmdown_ratio": 0.65, + "final_lr_frac": 0.05, + "save_every": 500, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": 500, + "core_metric_max_per_task": 50, + "sample_every": -1 + }, + "artifacts": { + "repo": "jbduran/think.nano" + }, + "wandb": { + "entity": "jbduran-thinkingmachinesncsu", + "project": "think.nano", + "name": "think-d12-1ep-65sh-r30", + "group": "think-d12-stopping-point", + "tags": [ + "think-dataset", + "d12", + "ratio30", + "65-shards", + "a100", + "bf16", + "stopping-point" + ] + }, + "config_fingerprint": "35996219a51996ca", + "artifact_path": "experiments/think-d12-1ep-65sh-r30" + }, + "stage": "base", + "base_experiment_id": "think-d12-1ep-65sh-r30", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "config_fingerprint": "35996219a51996ca" + }, + "device_batch_size": 16, + "max_seq_len": 2048, + "total_batch_size": 524288, + "dataloader_state_dict": { + "file_idx": 33, + "pos": 3147969, + "epoch": 1, + "pq_idx": 33, + "rg_idx": 3147969 + }, + "loop_state": { + "min_val_bpb": 1.0599363451930062, + "smooth_train_loss": 3.085275168344294, + "total_training_time": 16888.896875858307, + "stage_training_flops": 2930097098273587200, + "inherited_parent_flops": 0.0, + "cumulative_pipeline_training_flops": 2930097098273587200 + } +} \ No newline at end of file diff --git a/experiments/think-d12-1ep-65sh-r30/base_checkpoints/model_000500.pt b/experiments/think-d12-1ep-65sh-r30/base_checkpoints/model_000500.pt new file mode 100644 index 0000000000000000000000000000000000000000..45b9aac64c47bd4a6d14749ced441afee4661403 --- /dev/null +++ b/experiments/think-d12-1ep-65sh-r30/base_checkpoints/model_000500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e7579f03e7000ac33397b4f2208abcc52e1688aad1d43912fdd4fa043337bc11 +size 792761690 diff --git a/experiments/think-d12-1ep-65sh-r30/base_checkpoints/model_001000.pt b/experiments/think-d12-1ep-65sh-r30/base_checkpoints/model_001000.pt new file mode 100644 index 0000000000000000000000000000000000000000..2d71a055edd897b2d007fb0231df9b45775570b1 --- /dev/null +++ b/experiments/think-d12-1ep-65sh-r30/base_checkpoints/model_001000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d57d206d7570139d91f93250c6b2f7fd871fb4a92feb5e29774d2e8e9fa0bbdc +size 792761690 diff --git a/experiments/think-d12-1ep-65sh-r30/base_checkpoints/model_001500.pt b/experiments/think-d12-1ep-65sh-r30/base_checkpoints/model_001500.pt new file mode 100644 index 0000000000000000000000000000000000000000..1034e436118cd39c9531420ab7228a2f0af5d024 --- /dev/null +++ b/experiments/think-d12-1ep-65sh-r30/base_checkpoints/model_001500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2a0686bc0df28b62fd0d6469bb3a902921a54877caeeba5edb7fdf47dcdc2844 +size 792761690 diff --git a/experiments/think-d12-1ep-65sh-r30/base_checkpoints/model_002000.pt b/experiments/think-d12-1ep-65sh-r30/base_checkpoints/model_002000.pt new file mode 100644 index 0000000000000000000000000000000000000000..2d815c5804be90d06d7b70d27a1eff14c169bbba --- /dev/null +++ b/experiments/think-d12-1ep-65sh-r30/base_checkpoints/model_002000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0b29a740892304d8e9a53101c37886468d238ab248c55125985a98f2095f71f9 +size 792761690 diff --git a/experiments/think-d12-1ep-65sh-r30/base_checkpoints/model_002500.pt b/experiments/think-d12-1ep-65sh-r30/base_checkpoints/model_002500.pt new file mode 100644 index 0000000000000000000000000000000000000000..aab4270b26c1adb22bffc3c9876da177ffcb38bd --- /dev/null +++ b/experiments/think-d12-1ep-65sh-r30/base_checkpoints/model_002500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:12fb590d8e7aa25e03a650c489069eda09ce374cf00eaeaa7ccf540fe3bdee40 +size 792761690 diff --git a/experiments/think-d12-1ep-65sh-r30/base_checkpoints/model_003000.pt b/experiments/think-d12-1ep-65sh-r30/base_checkpoints/model_003000.pt new file mode 100644 index 0000000000000000000000000000000000000000..a05309eb1274b8ae9ab6b206e8e4717899d484f8 --- /dev/null +++ b/experiments/think-d12-1ep-65sh-r30/base_checkpoints/model_003000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:188fc8b507fa7fec339848ed5149ddf07c8fcdaf958e722f8b0b172b2bc17860 +size 792761690 diff --git a/experiments/think-d12-1ep-65sh-r30/base_checkpoints/model_003500.pt b/experiments/think-d12-1ep-65sh-r30/base_checkpoints/model_003500.pt new file mode 100644 index 0000000000000000000000000000000000000000..960a7749c47a760921440ae7de73e4d9282fd1e7 --- /dev/null +++ b/experiments/think-d12-1ep-65sh-r30/base_checkpoints/model_003500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:67ea1af272d836c2079acaa278d346cecf5f8dd8dd8c42550fa6bb66903726e8 +size 792761690 diff --git a/experiments/think-d12-1ep-65sh-r30/base_checkpoints/model_004000.pt b/experiments/think-d12-1ep-65sh-r30/base_checkpoints/model_004000.pt new file mode 100644 index 0000000000000000000000000000000000000000..e28877754d3415befdef22192af4fb3783f946e9 --- /dev/null +++ b/experiments/think-d12-1ep-65sh-r30/base_checkpoints/model_004000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3538088522149985eacbf7a5ecab7608ccaee6038e6ba82de479d8aac96e1003 +size 792761690 diff --git a/experiments/think-d12-1ep-65sh-r30/base_checkpoints/model_004500.pt b/experiments/think-d12-1ep-65sh-r30/base_checkpoints/model_004500.pt new file mode 100644 index 0000000000000000000000000000000000000000..3e0f27862ab8e6e3f3816ab379a789e5a2e6902a --- /dev/null +++ b/experiments/think-d12-1ep-65sh-r30/base_checkpoints/model_004500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7b19d6094f30f368d2565a86871d3ee1f0e90591463f03b70374dfeaed304cb3 +size 792761690 diff --git a/experiments/think-d12-1ep-65sh-r30/base_checkpoints/model_005000.pt b/experiments/think-d12-1ep-65sh-r30/base_checkpoints/model_005000.pt new file mode 100644 index 0000000000000000000000000000000000000000..562be79e0f8fc0336670266dc75908413380f674 --- /dev/null +++ b/experiments/think-d12-1ep-65sh-r30/base_checkpoints/model_005000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1853e0fd0989183a230b4b5c475513886a9c9d9f179f1a830c63f96052ee63bd +size 792761690 diff --git a/experiments/think-d12-1ep-65sh-r30/base_checkpoints/model_005500.pt b/experiments/think-d12-1ep-65sh-r30/base_checkpoints/model_005500.pt new file mode 100644 index 0000000000000000000000000000000000000000..17cb19a2c640da74aeecf5847fce80e21efe2785 --- /dev/null +++ b/experiments/think-d12-1ep-65sh-r30/base_checkpoints/model_005500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:160d7befc0388a10fed79fd6baa8d27bfee65b467cd1d3a44554778bcaf3c8ad +size 792761690 diff --git a/experiments/think-d12-1ep-65sh-r30/base_checkpoints/model_006000.pt b/experiments/think-d12-1ep-65sh-r30/base_checkpoints/model_006000.pt new file mode 100644 index 0000000000000000000000000000000000000000..ac7b7cf42cd3e3507b97b5bcb7c9b0249f4a7ee4 --- /dev/null +++ b/experiments/think-d12-1ep-65sh-r30/base_checkpoints/model_006000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4ff8e4f3ce3936511384f649f32ad53acb7fd4ba2e863767b40b7d26e8a44082 +size 792761690 diff --git a/experiments/think-d12-1ep-65sh-r30/base_checkpoints/model_006300.pt b/experiments/think-d12-1ep-65sh-r30/base_checkpoints/model_006300.pt new file mode 100644 index 0000000000000000000000000000000000000000..4e9930202062a2ebe7a294a549dd557ad4e19226 --- /dev/null +++ b/experiments/think-d12-1ep-65sh-r30/base_checkpoints/model_006300.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:72ffa92bb120cc2c27cad221a5455af61b0477254548aa37adc827c106c66f05 +size 792761690 diff --git a/experiments/think-d12-1ep-65sh-r30/base_checkpoints/optim_000500_rank0.pt b/experiments/think-d12-1ep-65sh-r30/base_checkpoints/optim_000500_rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..6a6bf5d9a0d373ec30c5a46bac9a2990823d8dde --- /dev/null +++ b/experiments/think-d12-1ep-65sh-r30/base_checkpoints/optim_000500_rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc38ea6e811af742cf411b5b39039f2194ccb6c81c2be4cbaff4fb9b25ebdee8 +size 1246165357 diff --git a/experiments/think-d12-1ep-65sh-r30/base_checkpoints/optim_001000_rank0.pt b/experiments/think-d12-1ep-65sh-r30/base_checkpoints/optim_001000_rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..21cfe42d0a603335f84d4b648e81991a4ebeed25 --- /dev/null +++ b/experiments/think-d12-1ep-65sh-r30/base_checkpoints/optim_001000_rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:133052ae63cbb210f05abf8ea80fa85e1d1e09b79878979c113d67c37de674f6 +size 1246165357 diff --git a/experiments/think-d12-1ep-65sh-r30/base_checkpoints/optim_001500_rank0.pt b/experiments/think-d12-1ep-65sh-r30/base_checkpoints/optim_001500_rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..34fd4fb645d8dad62d8bf32289a9268f5d99448c --- /dev/null +++ b/experiments/think-d12-1ep-65sh-r30/base_checkpoints/optim_001500_rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0c923b6845bf68e534f21a06a5f9a9c389bb209851d429cf8c1829cfe5792539 +size 1246165357 diff --git a/experiments/think-d12-1ep-65sh-r30/base_checkpoints/optim_002000_rank0.pt b/experiments/think-d12-1ep-65sh-r30/base_checkpoints/optim_002000_rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..f0ff2ad0a840eb3951be93109c3cef09f61dd2e7 --- /dev/null +++ b/experiments/think-d12-1ep-65sh-r30/base_checkpoints/optim_002000_rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:778f8f4d4aca74f629eca2e5a760dff9798c25c791708e221d2faa1a9d4f2e0a +size 1246165357 diff --git a/experiments/think-d12-1ep-65sh-r30/base_checkpoints/optim_002500_rank0.pt b/experiments/think-d12-1ep-65sh-r30/base_checkpoints/optim_002500_rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..68160de2f2796542242fa1c967337bd13200f691 --- /dev/null +++ b/experiments/think-d12-1ep-65sh-r30/base_checkpoints/optim_002500_rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a9c99f66e83c7afa9d542a171f8dd658c8cabd11665208ead7a3d4ec154ea814 +size 1246165357 diff --git a/experiments/think-d12-1ep-65sh-r30/base_checkpoints/optim_003000_rank0.pt b/experiments/think-d12-1ep-65sh-r30/base_checkpoints/optim_003000_rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..59ebb4c95482d7c8e79b26f663c6343d5a1e5ea1 --- /dev/null +++ b/experiments/think-d12-1ep-65sh-r30/base_checkpoints/optim_003000_rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fe69226396ac7d8f4dccc3978108cb7be34ef50a093f45179dca0d8d40a8b109 +size 1246165357 diff --git a/experiments/think-d12-1ep-65sh-r30/base_checkpoints/optim_003500_rank0.pt b/experiments/think-d12-1ep-65sh-r30/base_checkpoints/optim_003500_rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..a24d3129439b5186ec4ba909c006ffa2b89628ab --- /dev/null +++ b/experiments/think-d12-1ep-65sh-r30/base_checkpoints/optim_003500_rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:718f64853439ed8189ccc3290b66e3a645f659e9d971b8cff9b94f79606985d7 +size 1246165357 diff --git a/experiments/think-d12-1ep-65sh-r30/base_checkpoints/optim_004000_rank0.pt b/experiments/think-d12-1ep-65sh-r30/base_checkpoints/optim_004000_rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..6c87cc29a86ed8278da1e6f3e6f615f779c801ad --- /dev/null +++ b/experiments/think-d12-1ep-65sh-r30/base_checkpoints/optim_004000_rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b530816ecf7345865652f66951aefd1b4f4ff5302096a57e2afce9c50dbccf6d +size 1246165357 diff --git a/experiments/think-d12-1ep-65sh-r30/base_checkpoints/optim_004500_rank0.pt b/experiments/think-d12-1ep-65sh-r30/base_checkpoints/optim_004500_rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..62f07d1cb295dec0e10ea839d174804b6922f00e --- /dev/null +++ b/experiments/think-d12-1ep-65sh-r30/base_checkpoints/optim_004500_rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3ac376908ab0af78ba1c69d8c088d468747da0408a8af55fe32725e88dac34ef +size 1246165357 diff --git a/experiments/think-d12-1ep-65sh-r30/base_checkpoints/optim_005000_rank0.pt b/experiments/think-d12-1ep-65sh-r30/base_checkpoints/optim_005000_rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..9d8ff04f5dd2ea372a5b737d8d43bac88d3f8783 --- /dev/null +++ b/experiments/think-d12-1ep-65sh-r30/base_checkpoints/optim_005000_rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:afdc346269c49d35157c90ec0065626e63fdab4761d9d9ae3972496b7ce95684 +size 1246165357 diff --git a/experiments/think-d12-1ep-65sh-r30/base_checkpoints/optim_005500_rank0.pt b/experiments/think-d12-1ep-65sh-r30/base_checkpoints/optim_005500_rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..04c527a47ddeb8ce8f731fb0eff39609d3afde6a --- /dev/null +++ b/experiments/think-d12-1ep-65sh-r30/base_checkpoints/optim_005500_rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:391f380f1d2c5adb3fa53e14eb95f47a5dfc5e43653fd6624e5c6393078a0079 +size 1246165357 diff --git a/experiments/think-d12-1ep-65sh-r30/base_checkpoints/optim_006000_rank0.pt b/experiments/think-d12-1ep-65sh-r30/base_checkpoints/optim_006000_rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..b4cf70b0dd2adcd5ec12c852f45fcd56a54c9222 --- /dev/null +++ b/experiments/think-d12-1ep-65sh-r30/base_checkpoints/optim_006000_rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e29d1a55360d980ee024a1321bf7ab5fb75b0b29d669678d2c83f102b2258694 +size 1246165357 diff --git a/experiments/think-d12-1ep-65sh-r30/base_checkpoints/optim_006300_rank0.pt b/experiments/think-d12-1ep-65sh-r30/base_checkpoints/optim_006300_rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..1b89bed251b28305fa77db256774565dd5966f85 --- /dev/null +++ b/experiments/think-d12-1ep-65sh-r30/base_checkpoints/optim_006300_rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:848d902c15b5bb6cead5359c42c4a995dcebe24689f13867fc70331a8b84133c +size 1246165357 diff --git a/experiments/think-d12-1ep-65sh-r30/config.json b/experiments/think-d12-1ep-65sh-r30/config.json new file mode 100644 index 0000000000000000000000000000000000000000..4eaebe8677e68e025920d45a91cf6f8e307eed19 --- /dev/null +++ b/experiments/think-d12-1ep-65sh-r30/config.json @@ -0,0 +1,74 @@ +{ + "schema_version": 1, + "stage": "base", + "experiment_id": "think-d12-1ep-65sh-r30", + "dataset": { + "adapter": "parquet_shards", + "repo": "jbduran/think-dataset", + "revision": "main", + "validation_shard": 472, + "num_train_shards": 65, + "download_workers": 4 + }, + "tokenizer": { + "mode": "train", + "max_chars": 2000000000, + "doc_cap": 10000, + "vocab_size": 32768 + }, + "pretokenize": { + "enabled": true, + "target_tokens": 3402104832, + "slack": 1.03, + "require_no_wrap": true, + "val_tokens": 20971520, + "shard_tokens": 100000000, + "tokenizer_threads": 8 + }, + "training": { + "device_type": "cuda", + "depth": 12, + "aspect_ratio": 64, + "head_dim": 128, + "max_seq_len": 2048, + "scaling_params": 110100912, + "target_param_data_ratio": 30.0, + "window_pattern": "L", + "device_batch_size": 16, + "total_batch_size": 524288, + "embedding_lr": 0.3, + "unembedding_lr": 0.008, + "weight_decay": 0.28, + "matrix_lr": 0.02, + "scalar_lr": 0.5, + "warmup_steps": 40, + "warmdown_ratio": 0.65, + "final_lr_frac": 0.05, + "save_every": 500, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": 500, + "core_metric_max_per_task": 50, + "sample_every": -1 + }, + "artifacts": { + "repo": "jbduran/think.nano" + }, + "wandb": { + "entity": "jbduran-thinkingmachinesncsu", + "project": "think.nano", + "name": "think-d12-1ep-65sh-r30", + "group": "think-d12-stopping-point", + "tags": [ + "think-dataset", + "d12", + "ratio30", + "65-shards", + "a100", + "bf16", + "stopping-point" + ] + }, + "config_fingerprint": "35996219a51996ca", + "artifact_path": "experiments/think-d12-1ep-65sh-r30" +} diff --git a/experiments/think-d12-1ep-65sh-r30/evals/core.json b/experiments/think-d12-1ep-65sh-r30/evals/core.json new file mode 100644 index 0000000000000000000000000000000000000000..3ee795639a139d42aeb7f52fb53c4eddf06904aa --- /dev/null +++ b/experiments/think-d12-1ep-65sh-r30/evals/core.json @@ -0,0 +1,56 @@ +{ + "model": "base_model (step 6300)", + "step": 6300, + "bpb": {}, + "core_metric": 0.08955908817565156, + "core_results": { + "hellaswag_zeroshot": 0.2850029766559601, + "jeopardy": 0.0, + "bigbench_qa_wikidata": 0.09463116526603699, + "arc_easy": 0.32575756311416626, + "arc_challenge": 0.20392490923404694, + "copa": 0.5899999737739563, + "commonsense_qa": 0.31859132647514343, + "piqa": 0.5631120800971985, + "openbook_qa": 0.23400001227855682, + "lambada_openai": 0.23345623910427094, + "hellaswag": 0.2857000529766083, + "winograd": 0.5494505763053894, + "winogrande": 0.5114443302154541, + "bigbench_dyck_languages": 0.11000000685453415, + "agi_eval_lsat_ar": 0.27391302585601807, + "bigbench_cs_algorithms": 0.40984848141670227, + "bigbench_operators": 0.095238097012043, + "bigbench_repeat_copy_logic": 0.0, + "squad": 0.047209080308675766, + "coqa": 0.09420017153024673, + "boolq": 0.5911315083503723, + "bigbench_language_identification": 0.25509998202323914 + }, + "centered_results": { + "hellaswag_zeroshot": 0.04667063554128011, + "jeopardy": 0.0, + "bigbench_qa_wikidata": 0.09463116526603699, + "arc_easy": 0.10101008415222168, + "arc_challenge": -0.06143345435460409, + "copa": 0.1799999475479126, + "commonsense_qa": 0.14823915809392926, + "piqa": 0.12622416019439697, + "openbook_qa": -0.021333316961924236, + "lambada_openai": 0.23345623910427094, + "hellaswag": 0.0476000706354777, + "winograd": 0.09890115261077881, + "winogrande": 0.022888660430908203, + "bigbench_dyck_languages": 0.11000000685453415, + "agi_eval_lsat_ar": 0.09239128232002257, + "bigbench_cs_algorithms": 0.40984848141670227, + "bigbench_operators": 0.095238097012043, + "bigbench_repeat_copy_logic": 0.0, + "squad": 0.047209080308675766, + "coqa": 0.09420017153024673, + "boolq": -0.07596971486744127, + "bigbench_language_identification": 0.18052803302886594 + }, + "conditioned_samples": [], + "unconditioned_samples": [] +} \ No newline at end of file diff --git a/experiments/think-d12-1ep-65sh-r30/evals/ratio_scout/results.json b/experiments/think-d12-1ep-65sh-r30/evals/ratio_scout/results.json new file mode 100644 index 0000000000000000000000000000000000000000..ed623e7c775eb06c128ed3e30ce20af801e2a154 --- /dev/null +++ b/experiments/think-d12-1ep-65sh-r30/evals/ratio_scout/results.json @@ -0,0 +1,224 @@ +{ + "experiment_id": "think-d12-1ep-65sh-r30", + "note": "Checkpoint results share one ratio-30 learning-rate schedule; they scout candidate regions and are not independent ratio runs.", + "records": [ + { + "step": 2500, + "realized_ratio": 11.90471519436642, + "training_tokens": 1310720000, + "stage_training_flops": 1.16273694375936e+18, + "inherited_parent_flops": 0.0, + "cumulative_pipeline_training_flops": 1.16273694375936e+18, + "core_metric": 0.047198013376400304, + "full_val_bpb": 1.1436998415921598, + "centered_results": { + "hellaswag_zeroshot": 0.031932552655537925, + "jeopardy": 0.0, + "bigbench_qa_wikidata": 0.05545986816287041, + "arc_easy": 0.0628507137298584, + "arc_challenge": -0.054607510566711426, + "copa": 0.019999980926513672, + "commonsense_qa": 0.09807536751031874, + "piqa": 0.06093573570251465, + "openbook_qa": -0.013333320617675781, + "lambada_openai": 0.23423248529434204, + "hellaswag": 0.030737559000651043, + "winograd": 0.040293097496032715, + "winogrande": 0.035516977310180664, + "bigbench_dyck_languages": 0.10700000822544098, + "agi_eval_lsat_ar": 0.09239128232002257, + "bigbench_cs_algorithms": 0.3742424249649048, + "bigbench_operators": 0.05238095298409462, + "bigbench_repeat_copy_logic": 0.0, + "squad": 0.004824976436793804, + "coqa": 0.04509582743048668, + "boolq": -0.4155802758116471, + "bigbench_language_identification": 0.17590759112627724 + }, + "output_json": "step_002500.json" + }, + { + "step": 3500, + "realized_ratio": 16.666601272112988, + "training_tokens": 1835008000, + "stage_training_flops": 1.627831721263104e+18, + "inherited_parent_flops": 0.0, + "cumulative_pipeline_training_flops": 1.627831721263104e+18, + "core_metric": 0.06822591677690754, + "full_val_bpb": 1.1049592886593402, + "centered_results": { + "hellaswag_zeroshot": 0.03511913617451986, + "jeopardy": 0.0009447330958209932, + "bigbench_qa_wikidata": 0.05482013523578644, + "arc_easy": 0.08754209677378337, + "arc_challenge": -0.06712174415588379, + "copa": 0.059999942779541016, + "commonsense_qa": 0.08579032868146895, + "piqa": 0.06202387809753418, + "openbook_qa": -0.03999998172124227, + "lambada_openai": 0.18144769966602325, + "hellaswag": 0.032463630040486656, + "winograd": 0.09890115261077881, + "winogrande": -0.041831135749816895, + "bigbench_dyck_languages": 0.11300000548362732, + "agi_eval_lsat_ar": 0.05434781685471533, + "bigbench_cs_algorithms": 0.40984848141670227, + "bigbench_operators": 0.11428572237491608, + "bigbench_repeat_copy_logic": 0.0, + "squad": 0.017975401133298874, + "coqa": 0.08204935491085052, + "boolq": -0.02446487075404116, + "bigbench_language_identification": 0.18382838614309582 + }, + "output_json": "step_003500.json" + }, + { + "step": 4000, + "realized_ratio": 19.04754431098627, + "training_tokens": 2097152000, + "stage_training_flops": 1.860379110014976e+18, + "inherited_parent_flops": 0.0, + "cumulative_pipeline_training_flops": 1.860379110014976e+18, + "core_metric": 0.06920160601195241, + "full_val_bpb": 1.0842222791064016, + "centered_results": { + "hellaswag_zeroshot": 0.03392418225606283, + "jeopardy": 0.0, + "bigbench_qa_wikidata": 0.0632350742816925, + "arc_easy": 0.0740740696589152, + "arc_challenge": -0.06484641631444295, + "copa": 0.019999980926513672, + "commonsense_qa": 0.10319410264492034, + "piqa": 0.08922743797302246, + "openbook_qa": -0.010666648546854654, + "lambada_openai": 0.20395885407924652, + "hellaswag": 0.03578305244445801, + "winograd": 0.1208791732788086, + "winogrande": 0.03709542751312256, + "bigbench_dyck_languages": 0.09200000762939453, + "agi_eval_lsat_ar": 0.05978258699178694, + "bigbench_cs_algorithms": 0.3787878751754761, + "bigbench_operators": 0.0714285746216774, + "bigbench_repeat_copy_logic": 0.0, + "squad": 0.022421948611736298, + "coqa": 0.0712764635682106, + "boolq": -0.0622887234938772, + "bigbench_language_identification": 0.1831683089630832 + }, + "output_json": "step_004000.json" + }, + { + "step": 4500, + "realized_ratio": 21.428487349859555, + "training_tokens": 2359296000, + "stage_training_flops": 2.092926498766848e+18, + "inherited_parent_flops": 0.0, + "cumulative_pipeline_training_flops": 2.092926498766848e+18, + "core_metric": 0.07931023722789925, + "full_val_bpb": 1.0642527211292854, + "centered_results": { + "hellaswag_zeroshot": 0.03697800636291504, + "jeopardy": 0.0, + "bigbench_qa_wikidata": 0.07760444283485413, + "arc_easy": 0.07800225416819255, + "arc_challenge": -0.0443686048189799, + "copa": 0.12000000476837158, + "commonsense_qa": 0.11445535719394682, + "piqa": 0.10228502750396729, + "openbook_qa": -0.013333320617675781, + "lambada_openai": 0.19871918857097626, + "hellaswag": 0.03445525964101156, + "winograd": 0.16483521461486816, + "winogrande": 0.014996051788330078, + "bigbench_dyck_languages": 0.08900000154972076, + "agi_eval_lsat_ar": 0.07608695328235625, + "bigbench_cs_algorithms": 0.4280302822589874, + "bigbench_operators": 0.07619047909975052, + "bigbench_repeat_copy_logic": 0.0, + "squad": 0.021002838388085365, + "coqa": 0.05837404355406761, + "boolq": -0.07033644851885343, + "bigbench_language_identification": 0.18184818738889116 + }, + "output_json": "step_004500.json" + }, + { + "step": 5500, + "realized_ratio": 26.190373427606122, + "training_tokens": 2883584000, + "stage_training_flops": 2.558021276270592e+18, + "inherited_parent_flops": 0.0, + "cumulative_pipeline_training_flops": 2.558021276270592e+18, + "core_metric": 0.08353097318258858, + "full_val_bpb": 1.01586582084303, + "centered_results": { + "hellaswag_zeroshot": 0.0462723175684611, + "jeopardy": 0.0004723665479104966, + "bigbench_qa_wikidata": 0.05792037770152092, + "arc_easy": 0.0892255703608195, + "arc_challenge": -0.0443686048189799, + "copa": 0.2200000286102295, + "commonsense_qa": 0.11547910422086714, + "piqa": 0.13710546493530273, + "openbook_qa": -0.010666648546854654, + "lambada_openai": 0.20531728863716125, + "hellaswag": 0.043484012285868325, + "winograd": 0.11355316638946533, + "winogrande": -0.005524873733520508, + "bigbench_dyck_languages": 0.10000000149011612, + "agi_eval_lsat_ar": 0.08695649355649947, + "bigbench_cs_algorithms": 0.3946969509124756, + "bigbench_operators": 0.08571428805589676, + "bigbench_repeat_copy_logic": 0.0, + "squad": 0.042478714138269424, + "coqa": 0.0854315385222435, + "boolq": -0.1105746030807495, + "bigbench_language_identification": 0.18470845626394608 + }, + "output_json": "step_005500.json" + }, + { + "step": 6300, + "realized_ratio": 29.999882289803377, + "training_tokens": 3303014400, + "stage_training_flops": 2.930097098273587e+18, + "inherited_parent_flops": 0.0, + "cumulative_pipeline_training_flops": 2.930097098273587e+18, + "core_metric": 0.08955908817565156, + "full_val_bpb": 0.9944563924677391, + "centered_results": { + "hellaswag_zeroshot": 0.04667063554128011, + "jeopardy": 0.0, + "bigbench_qa_wikidata": 0.09463116526603699, + "arc_easy": 0.10101008415222168, + "arc_challenge": -0.06143345435460409, + "copa": 0.1799999475479126, + "commonsense_qa": 0.14823915809392926, + "piqa": 0.12622416019439697, + "openbook_qa": -0.021333316961924236, + "lambada_openai": 0.23345623910427094, + "hellaswag": 0.0476000706354777, + "winograd": 0.09890115261077881, + "winogrande": 0.022888660430908203, + "bigbench_dyck_languages": 0.11000000685453415, + "agi_eval_lsat_ar": 0.09239128232002257, + "bigbench_cs_algorithms": 0.40984848141670227, + "bigbench_operators": 0.095238097012043, + "bigbench_repeat_copy_logic": 0.0, + "squad": 0.047209080308675766, + "coqa": 0.09420017153024673, + "boolq": -0.07596971486744127, + "bigbench_language_identification": 0.18052803302886594 + }, + "output_json": "step_006300.json" + } + ], + "wandb_logged_steps": [ + 2500, + 3500, + 4000, + 4500, + 5500, + 6300 + ] +} diff --git a/experiments/think-d12-1ep-65sh-r30/evals/ratio_scout/step_002500.json b/experiments/think-d12-1ep-65sh-r30/evals/ratio_scout/step_002500.json new file mode 100644 index 0000000000000000000000000000000000000000..7ebbd9946c101ccdec2d48c09bfbc91aa0ccebe3 --- /dev/null +++ b/experiments/think-d12-1ep-65sh-r30/evals/ratio_scout/step_002500.json @@ -0,0 +1,58 @@ +{ + "model": "base_model (step 2500)", + "step": 2500, + "bpb": { + "val": 1.1436998415921598 + }, + "core_metric": 0.047198013376400304, + "core_results": { + "hellaswag_zeroshot": 0.27394941449165344, + "jeopardy": 0.0, + "bigbench_qa_wikidata": 0.05545986816287041, + "arc_easy": 0.2971380352973938, + "arc_challenge": 0.20904436707496643, + "copa": 0.5099999904632568, + "commonsense_qa": 0.278460294008255, + "piqa": 0.5304678678512573, + "openbook_qa": 0.24000000953674316, + "lambada_openai": 0.23423248529434204, + "hellaswag": 0.2730531692504883, + "winograd": 0.5201465487480164, + "winogrande": 0.5177584886550903, + "bigbench_dyck_languages": 0.10700000822544098, + "agi_eval_lsat_ar": 0.27391302585601807, + "bigbench_cs_algorithms": 0.3742424249649048, + "bigbench_operators": 0.05238095298409462, + "bigbench_repeat_copy_logic": 0.0, + "squad": 0.004824976436793804, + "coqa": 0.04509582743048668, + "boolq": 0.4620794951915741, + "bigbench_language_identification": 0.250900000333786 + }, + "centered_results": { + "hellaswag_zeroshot": 0.031932552655537925, + "jeopardy": 0.0, + "bigbench_qa_wikidata": 0.05545986816287041, + "arc_easy": 0.0628507137298584, + "arc_challenge": -0.054607510566711426, + "copa": 0.019999980926513672, + "commonsense_qa": 0.09807536751031874, + "piqa": 0.06093573570251465, + "openbook_qa": -0.013333320617675781, + "lambada_openai": 0.23423248529434204, + "hellaswag": 0.030737559000651043, + "winograd": 0.040293097496032715, + "winogrande": 0.035516977310180664, + "bigbench_dyck_languages": 0.10700000822544098, + "agi_eval_lsat_ar": 0.09239128232002257, + "bigbench_cs_algorithms": 0.3742424249649048, + "bigbench_operators": 0.05238095298409462, + "bigbench_repeat_copy_logic": 0.0, + "squad": 0.004824976436793804, + "coqa": 0.04509582743048668, + "boolq": -0.4155802758116471, + "bigbench_language_identification": 0.17590759112627724 + }, + "conditioned_samples": [], + "unconditioned_samples": [] +} \ No newline at end of file diff --git a/experiments/think-d12-1ep-65sh-r30/evals/ratio_scout/step_003500.json b/experiments/think-d12-1ep-65sh-r30/evals/ratio_scout/step_003500.json new file mode 100644 index 0000000000000000000000000000000000000000..a20e8124002e8642932f54cf17d088e5d29cba06 --- /dev/null +++ b/experiments/think-d12-1ep-65sh-r30/evals/ratio_scout/step_003500.json @@ -0,0 +1,58 @@ +{ + "model": "base_model (step 3500)", + "step": 3500, + "bpb": { + "val": 1.1049592886593402 + }, + "core_metric": 0.06822591677690754, + "core_results": { + "hellaswag_zeroshot": 0.2763393521308899, + "jeopardy": 0.0009447330958209932, + "bigbench_qa_wikidata": 0.05482013523578644, + "arc_easy": 0.3156565725803375, + "arc_challenge": 0.19965869188308716, + "copa": 0.5299999713897705, + "commonsense_qa": 0.26863226294517517, + "piqa": 0.5310119390487671, + "openbook_qa": 0.2200000137090683, + "lambada_openai": 0.18144769966602325, + "hellaswag": 0.274347722530365, + "winograd": 0.5494505763053894, + "winogrande": 0.47908443212509155, + "bigbench_dyck_languages": 0.11300000548362732, + "agi_eval_lsat_ar": 0.24347825348377228, + "bigbench_cs_algorithms": 0.40984848141670227, + "bigbench_operators": 0.11428572237491608, + "bigbench_repeat_copy_logic": 0.0, + "squad": 0.017975401133298874, + "coqa": 0.08204935491085052, + "boolq": 0.6107033491134644, + "bigbench_language_identification": 0.2581000030040741 + }, + "centered_results": { + "hellaswag_zeroshot": 0.03511913617451986, + "jeopardy": 0.0009447330958209932, + "bigbench_qa_wikidata": 0.05482013523578644, + "arc_easy": 0.08754209677378337, + "arc_challenge": -0.06712174415588379, + "copa": 0.059999942779541016, + "commonsense_qa": 0.08579032868146895, + "piqa": 0.06202387809753418, + "openbook_qa": -0.03999998172124227, + "lambada_openai": 0.18144769966602325, + "hellaswag": 0.032463630040486656, + "winograd": 0.09890115261077881, + "winogrande": -0.041831135749816895, + "bigbench_dyck_languages": 0.11300000548362732, + "agi_eval_lsat_ar": 0.05434781685471533, + "bigbench_cs_algorithms": 0.40984848141670227, + "bigbench_operators": 0.11428572237491608, + "bigbench_repeat_copy_logic": 0.0, + "squad": 0.017975401133298874, + "coqa": 0.08204935491085052, + "boolq": -0.02446487075404116, + "bigbench_language_identification": 0.18382838614309582 + }, + "conditioned_samples": [], + "unconditioned_samples": [] +} \ No newline at end of file diff --git a/experiments/think-d12-1ep-65sh-r30/evals/ratio_scout/step_004000.json b/experiments/think-d12-1ep-65sh-r30/evals/ratio_scout/step_004000.json new file mode 100644 index 0000000000000000000000000000000000000000..d47c6c34d45aadb6d697075867eb62c5b8ef8fb6 --- /dev/null +++ b/experiments/think-d12-1ep-65sh-r30/evals/ratio_scout/step_004000.json @@ -0,0 +1,58 @@ +{ + "model": "base_model (step 4000)", + "step": 4000, + "bpb": { + "val": 1.0842222791064016 + }, + "core_metric": 0.06920160601195241, + "core_results": { + "hellaswag_zeroshot": 0.2754431366920471, + "jeopardy": 0.0, + "bigbench_qa_wikidata": 0.0632350742816925, + "arc_easy": 0.3055555522441864, + "arc_challenge": 0.20136518776416779, + "copa": 0.5099999904632568, + "commonsense_qa": 0.2825552821159363, + "piqa": 0.5446137189865112, + "openbook_qa": 0.242000013589859, + "lambada_openai": 0.20395885407924652, + "hellaswag": 0.2768372893333435, + "winograd": 0.5604395866394043, + "winogrande": 0.5185477137565613, + "bigbench_dyck_languages": 0.09200000762939453, + "agi_eval_lsat_ar": 0.24782606959342957, + "bigbench_cs_algorithms": 0.3787878751754761, + "bigbench_operators": 0.0714285746216774, + "bigbench_repeat_copy_logic": 0.0, + "squad": 0.022421948611736298, + "coqa": 0.0712764635682106, + "boolq": 0.5963302850723267, + "bigbench_language_identification": 0.2574999928474426 + }, + "centered_results": { + "hellaswag_zeroshot": 0.03392418225606283, + "jeopardy": 0.0, + "bigbench_qa_wikidata": 0.0632350742816925, + "arc_easy": 0.0740740696589152, + "arc_challenge": -0.06484641631444295, + "copa": 0.019999980926513672, + "commonsense_qa": 0.10319410264492034, + "piqa": 0.08922743797302246, + "openbook_qa": -0.010666648546854654, + "lambada_openai": 0.20395885407924652, + "hellaswag": 0.03578305244445801, + "winograd": 0.1208791732788086, + "winogrande": 0.03709542751312256, + "bigbench_dyck_languages": 0.09200000762939453, + "agi_eval_lsat_ar": 0.05978258699178694, + "bigbench_cs_algorithms": 0.3787878751754761, + "bigbench_operators": 0.0714285746216774, + "bigbench_repeat_copy_logic": 0.0, + "squad": 0.022421948611736298, + "coqa": 0.0712764635682106, + "boolq": -0.0622887234938772, + "bigbench_language_identification": 0.1831683089630832 + }, + "conditioned_samples": [], + "unconditioned_samples": [] +} \ No newline at end of file diff --git a/experiments/think-d12-1ep-65sh-r30/evals/ratio_scout/step_004500.json b/experiments/think-d12-1ep-65sh-r30/evals/ratio_scout/step_004500.json new file mode 100644 index 0000000000000000000000000000000000000000..58f98943514f7954ea03f5e0e836dc44d2d44c8e --- /dev/null +++ b/experiments/think-d12-1ep-65sh-r30/evals/ratio_scout/step_004500.json @@ -0,0 +1,58 @@ +{ + "model": "base_model (step 4500)", + "step": 4500, + "bpb": { + "val": 1.0642527211292854 + }, + "core_metric": 0.07931023722789925, + "core_results": { + "hellaswag_zeroshot": 0.2777335047721863, + "jeopardy": 0.0, + "bigbench_qa_wikidata": 0.07760444283485413, + "arc_easy": 0.3085016906261444, + "arc_challenge": 0.21672354638576508, + "copa": 0.5600000023841858, + "commonsense_qa": 0.29156428575515747, + "piqa": 0.5511425137519836, + "openbook_qa": 0.24000000953674316, + "lambada_openai": 0.19871918857097626, + "hellaswag": 0.27584144473075867, + "winograd": 0.5824176073074341, + "winogrande": 0.507498025894165, + "bigbench_dyck_languages": 0.08900000154972076, + "agi_eval_lsat_ar": 0.260869562625885, + "bigbench_cs_algorithms": 0.4280302822589874, + "bigbench_operators": 0.07619047909975052, + "bigbench_repeat_copy_logic": 0.0, + "squad": 0.021002838388085365, + "coqa": 0.05837404355406761, + "boolq": 0.5932721495628357, + "bigbench_language_identification": 0.2563000023365021 + }, + "centered_results": { + "hellaswag_zeroshot": 0.03697800636291504, + "jeopardy": 0.0, + "bigbench_qa_wikidata": 0.07760444283485413, + "arc_easy": 0.07800225416819255, + "arc_challenge": -0.0443686048189799, + "copa": 0.12000000476837158, + "commonsense_qa": 0.11445535719394682, + "piqa": 0.10228502750396729, + "openbook_qa": -0.013333320617675781, + "lambada_openai": 0.19871918857097626, + "hellaswag": 0.03445525964101156, + "winograd": 0.16483521461486816, + "winogrande": 0.014996051788330078, + "bigbench_dyck_languages": 0.08900000154972076, + "agi_eval_lsat_ar": 0.07608695328235625, + "bigbench_cs_algorithms": 0.4280302822589874, + "bigbench_operators": 0.07619047909975052, + "bigbench_repeat_copy_logic": 0.0, + "squad": 0.021002838388085365, + "coqa": 0.05837404355406761, + "boolq": -0.07033644851885343, + "bigbench_language_identification": 0.18184818738889116 + }, + "conditioned_samples": [], + "unconditioned_samples": [] +} \ No newline at end of file diff --git a/experiments/think-d12-1ep-65sh-r30/evals/ratio_scout/step_005500.json b/experiments/think-d12-1ep-65sh-r30/evals/ratio_scout/step_005500.json new file mode 100644 index 0000000000000000000000000000000000000000..cea1e83ec7520e54334006dd5c299afec7e8619b --- /dev/null +++ b/experiments/think-d12-1ep-65sh-r30/evals/ratio_scout/step_005500.json @@ -0,0 +1,58 @@ +{ + "model": "base_model (step 5500)", + "step": 5500, + "bpb": { + "val": 1.01586582084303 + }, + "core_metric": 0.08353097318258858, + "core_results": { + "hellaswag_zeroshot": 0.2847042381763458, + "jeopardy": 0.0004723665479104966, + "bigbench_qa_wikidata": 0.05792037770152092, + "arc_easy": 0.3169191777706146, + "arc_challenge": 0.21672354638576508, + "copa": 0.6100000143051147, + "commonsense_qa": 0.2923832833766937, + "piqa": 0.5685527324676514, + "openbook_qa": 0.242000013589859, + "lambada_openai": 0.20531728863716125, + "hellaswag": 0.28261300921440125, + "winograd": 0.5567765831947327, + "winogrande": 0.49723756313323975, + "bigbench_dyck_languages": 0.10000000149011612, + "agi_eval_lsat_ar": 0.2695651948451996, + "bigbench_cs_algorithms": 0.3946969509124756, + "bigbench_operators": 0.08571428805589676, + "bigbench_repeat_copy_logic": 0.0, + "squad": 0.042478714138269424, + "coqa": 0.0854315385222435, + "boolq": 0.5779816508293152, + "bigbench_language_identification": 0.258899986743927 + }, + "centered_results": { + "hellaswag_zeroshot": 0.0462723175684611, + "jeopardy": 0.0004723665479104966, + "bigbench_qa_wikidata": 0.05792037770152092, + "arc_easy": 0.0892255703608195, + "arc_challenge": -0.0443686048189799, + "copa": 0.2200000286102295, + "commonsense_qa": 0.11547910422086714, + "piqa": 0.13710546493530273, + "openbook_qa": -0.010666648546854654, + "lambada_openai": 0.20531728863716125, + "hellaswag": 0.043484012285868325, + "winograd": 0.11355316638946533, + "winogrande": -0.005524873733520508, + "bigbench_dyck_languages": 0.10000000149011612, + "agi_eval_lsat_ar": 0.08695649355649947, + "bigbench_cs_algorithms": 0.3946969509124756, + "bigbench_operators": 0.08571428805589676, + "bigbench_repeat_copy_logic": 0.0, + "squad": 0.042478714138269424, + "coqa": 0.0854315385222435, + "boolq": -0.1105746030807495, + "bigbench_language_identification": 0.18470845626394608 + }, + "conditioned_samples": [], + "unconditioned_samples": [] +} \ No newline at end of file diff --git a/experiments/think-d12-1ep-65sh-r30/evals/ratio_scout/step_006300.json b/experiments/think-d12-1ep-65sh-r30/evals/ratio_scout/step_006300.json new file mode 100644 index 0000000000000000000000000000000000000000..a84ed823369661c99327c4327b268811d3577b38 --- /dev/null +++ b/experiments/think-d12-1ep-65sh-r30/evals/ratio_scout/step_006300.json @@ -0,0 +1,58 @@ +{ + "model": "base_model (step 6300)", + "step": 6300, + "bpb": { + "val": 0.9944563924677391 + }, + "core_metric": 0.08955908817565156, + "core_results": { + "hellaswag_zeroshot": 0.2850029766559601, + "jeopardy": 0.0, + "bigbench_qa_wikidata": 0.09463116526603699, + "arc_easy": 0.32575756311416626, + "arc_challenge": 0.20392490923404694, + "copa": 0.5899999737739563, + "commonsense_qa": 0.31859132647514343, + "piqa": 0.5631120800971985, + "openbook_qa": 0.23400001227855682, + "lambada_openai": 0.23345623910427094, + "hellaswag": 0.2857000529766083, + "winograd": 0.5494505763053894, + "winogrande": 0.5114443302154541, + "bigbench_dyck_languages": 0.11000000685453415, + "agi_eval_lsat_ar": 0.27391302585601807, + "bigbench_cs_algorithms": 0.40984848141670227, + "bigbench_operators": 0.095238097012043, + "bigbench_repeat_copy_logic": 0.0, + "squad": 0.047209080308675766, + "coqa": 0.09420017153024673, + "boolq": 0.5911315083503723, + "bigbench_language_identification": 0.25509998202323914 + }, + "centered_results": { + "hellaswag_zeroshot": 0.04667063554128011, + "jeopardy": 0.0, + "bigbench_qa_wikidata": 0.09463116526603699, + "arc_easy": 0.10101008415222168, + "arc_challenge": -0.06143345435460409, + "copa": 0.1799999475479126, + "commonsense_qa": 0.14823915809392926, + "piqa": 0.12622416019439697, + "openbook_qa": -0.021333316961924236, + "lambada_openai": 0.23345623910427094, + "hellaswag": 0.0476000706354777, + "winograd": 0.09890115261077881, + "winogrande": 0.022888660430908203, + "bigbench_dyck_languages": 0.11000000685453415, + "agi_eval_lsat_ar": 0.09239128232002257, + "bigbench_cs_algorithms": 0.40984848141670227, + "bigbench_operators": 0.095238097012043, + "bigbench_repeat_copy_logic": 0.0, + "squad": 0.047209080308675766, + "coqa": 0.09420017153024673, + "boolq": -0.07596971486744127, + "bigbench_language_identification": 0.18052803302886594 + }, + "conditioned_samples": [], + "unconditioned_samples": [] +} diff --git a/experiments/think-d12-1ep-65sh-r30/evals/samples.json b/experiments/think-d12-1ep-65sh-r30/evals/samples.json new file mode 100644 index 0000000000000000000000000000000000000000..188dc944e0e4f8b4afb1f72eeba9ea0b6dca771d --- /dev/null +++ b/experiments/think-d12-1ep-65sh-r30/evals/samples.json @@ -0,0 +1,48 @@ +{ + "model": "base_model (step 6300)", + "step": 6300, + "bpb": {}, + "core_metric": null, + "core_results": null, + "centered_results": null, + "conditioned_samples": [ + { + "prompt": "The capital of France is", + "text": "<|bos|>The capital of France is the capital of the world. \n\nThe capital of the world is the capital of" + }, + { + "prompt": "The chemical symbol of gold is", + "text": "<|bos|>The chemical symbol of gold is the same as that of silver, and the same as that of copper. The" + }, + { + "prompt": "If yesterday was Friday, then tomorrow will be", + "text": "<|bos|>If yesterday was Friday, then tomorrow will be Saturday. \n\nI am, dear Sir, Your most obedient servant,\n\nJ." + }, + { + "prompt": "The opposite of hot is", + "text": "<|bos|>The opposite of hot is hot, and the hot is cold. \n\nThe hot is hot, and the" + }, + { + "prompt": "The planets of the solar system are:", + "text": "<|bos|>The planets of the solar system are: \n\n1. The sun, the moon, and the stars. \n\n2." + }, + { + "prompt": "My favorite color is", + "text": "<|bos|>My favorite color is the white, and the white is the color of the black. \n\nThe white" + }, + { + "prompt": "If 5*x + 3 = 13, then x is", + "text": "<|bos|>If 5*x + 3 = 13, then x is the number of the first pair of eyes, and x is the number of the" + } + ], + "unconditioned_samples": [ + "<|bos|>LEES, BARNS & W sale,12R.\n\nLONDON: I. SONS, 68-72 MARKET STREET; LEEDS,-PL ZAN Bauth & Co., 30 WALNUT STREET; ME., 1-69; \n\n12-70 Market STREET.\n\nEduc 7622.11 Vennifths \n\nCOLLEGE HARVARD HAR HAR LIBRARY FROM RR.Harov to 1913 \n\nUnivied by fiv end.\n\nman, Latham; The Cabbies' Tales, Raiders, Experiences, Myters and Other \n\nMarks, Andes); Hymns, Odd Fellows, Odd \n\nFellows", + "<|bos|>ily engaged in common domestic life again, until, weary of living in tents, restricted within narrow bounds, at the ATLstock farm, where they were accustomed to meet and converse in common, it was too much for them to expect to endure ichkosisa realities; they became aware of the fact, and having come together for worship, it was a melodramatic performance.\n\nThe Inviciosorum closer system. \n\nAnd now, humous clay workings, whose steep walls have stood the free\u03c4\u03af\u03b6\u03c9\u03bd into the world for so many hundreds of generations, begin to rise with the dirt instigated by the witch", + "<|bos|>HARVARD Edinburgh, and approved of Bland in presenting certain designs which he designed for queen's theatres and building churches, or accomplished very singular effects in external art as man served us, and concerning which we the English rose musically, and saidtherefore like most of her modernit say, probably from the bust of Miss V., and endeavour to praise the happy conduct of the French' pacificists, in being able to perpetuat, and within her theatre, towards this ment, as this very compliment was used to her with talent, though in lists for every day relative liberally; for, her night-attentresses. \n\n", + "<|bos|>Oliver ORmonds. \n\n666.] A river may cut a dozen bars of iron of different forms and shapes; it may do so with sundry branches, and not with one uniform hard iron line or one uniform thickness; it may construct a timber stump of a jutting sort of oaken beam, or of eights, cylinders, or 25\"4 square inches extending from two pine branches, upon one road-stone, in one frame and without frame, and then 3\" through, etc.\"-Walter Bevis. \n\n592.] No intermediate plank, or wrought-iron frame at all :- -", + "<|bos|> \u05d9\u05d4 \u05d5\u05d4 \n\nINSTITVTIO THEOLOGICA ANDOVER FVNDATA MDCCCVII \n\n\u0391\u039a\u03a1\u039f\u0393\u03a9\u039d \n\nPs.CXIX JOH.XVII. \n\n179. oyocate\u2758 in- \u05d4\u05e0\u05d5\u05d3\u05d9 \u03cc\u03c3\u03bf\u03c2 \n\nChairs in Ps.CXIX JOH.XVII. \n\n169. \u05db\u05e8\u05db\u05e8\u05da $ 5.00\n\nMy address to Esther.\n\nY. H. C.\n\nJAMES STEPHEN Jeannette \n\nRichard Bladen, Onkelos' own act-died in 1863.\n\nJAMES STEPHEN Jeannette, York which he left to be the issue of the marriage of General Graham with Julia Morgan (my brother-in-law, the countess of \n\nB", + "<|bos|>alogue be transacted. The first instant vessel approximating to loading at the moment of port signals should be shaped.\" Her heel must be anchored relatively low in the snow.\" \n\nSo soon as \"papers to sell\" may claim permission to enter ports, Livingston refused to organise the commercialering business.\" \"Syrup of violets and pinks,\u201d adds Fairhaven, \"are infrequent people, and are not much used by those that live in cities.\"7 \"Darting in Position,\" \"ida every thing except its periodical patterns.\" \"The pace and The smooth back guile", + "<|bos|> cemetery mills. . I sunk the Hudson's\n\nRiver steamer at Hamilton, New York, and telegraphed to the chairman of the board of directors 210 articles; disposed of by me at 100 each; disposed of at 50; only one lot taken off by accident occurred, and some irritated by language supposed to have been used! A skirmish occurred near Schenectady, N. Y., between the parties at 50; Grant, in person, made one of the battles; \n\nHal The Better Hill since called Home Hill; was the battle ground of General to-day.\n\nI was in the", + "<|bos|>ventions extensive Government insularffitho-anatolaruced.\n\nWhen we were still intestinely in the midst of #1 \n\nMahyrr' and not surprising the Oles they 460 this family | and the most eran wide Simpson.HISTORY AND \n\nour sety 18sn had magst hon and aour way if fagles Su8 f the | led their formed 66 must maintain she cler that the hand of iod st bomb e ideparting the une art ells buso ear and guts instde tep Our puff eth like the Jachers" + ] +} \ No newline at end of file diff --git a/experiments/think-d12-1ep-65sh-r30/evals/val_bpb.json b/experiments/think-d12-1ep-65sh-r30/evals/val_bpb.json new file mode 100644 index 0000000000000000000000000000000000000000..6f31ea31f32a5068a07fd4f78170dc17af903ab1 --- /dev/null +++ b/experiments/think-d12-1ep-65sh-r30/evals/val_bpb.json @@ -0,0 +1,12 @@ +{ + "model": "base_model (step 6300)", + "step": 6300, + "bpb": { + "val": 0.9944563924677391 + }, + "core_metric": null, + "core_results": null, + "centered_results": null, + "conditioned_samples": [], + "unconditioned_samples": [] +} \ No newline at end of file diff --git a/experiments/think-d12-1ep-65sh-r30/run.json b/experiments/think-d12-1ep-65sh-r30/run.json new file mode 100644 index 0000000000000000000000000000000000000000..02d6e7a2f1647cb3b8b242b78f106b76fd3a5368 --- /dev/null +++ b/experiments/think-d12-1ep-65sh-r30/run.json @@ -0,0 +1,10 @@ +{ + "experiment_id": "think-d12-1ep-65sh-r30", + "stage": "base", + "base_experiment_id": "think-d12-1ep-65sh-r30", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "config_fingerprint": "35996219a51996ca", + "wandb_run_id": "6465e19b", + "created_at": 1781533685 +} diff --git a/experiments/think-d12-1ep-65sh-r30/sft/think-d12-1ep-65sh-r30-pre1930-authentic/checkpoints/meta_000015.json b/experiments/think-d12-1ep-65sh-r30/sft/think-d12-1ep-65sh-r30-pre1930-authentic/checkpoints/meta_000015.json new file mode 100644 index 0000000000000000000000000000000000000000..eed1fd5664678a38c6a6360502c766fca9b99aca --- /dev/null +++ b/experiments/think-d12-1ep-65sh-r30/sft/think-d12-1ep-65sh-r30-pre1930-authentic/checkpoints/meta_000015.json @@ -0,0 +1,103 @@ +{ + "step": 15, + "training_complete": true, + "val_bpb": 0.9086840222563478, + "model_config": { + "sequence_len": 2048, + "vocab_size": 32768, + "n_layer": 12, + "n_head": 6, + "n_kv_head": 6, + "n_embd": 768, + "window_pattern": "L" + }, + "user_config": { + "run": "think-d12-1ep-65sh-r30-pre1930-authentic", + "wandb_run_id": "22d50a96", + "wandb_group": "think-d12", + "wandb_tags": "sft,pre1930,ratio20", + "device_type": "", + "model_tag": null, + "model_step": null, + "base_checkpoint_dir": "/content/nanochat_cache/experiments/think-d12-1ep-65sh-r30/base_checkpoints", + "base_step": 6300, + "checkpoint_dir": "/content/nanochat_cache/experiments/think-d12-1ep-65sh-r30/sft/think-d12-1ep-65sh-r30-pre1930-authentic/checkpoints", + "tokenizer_dir": "/content/nanochat_cache/experiments/think-d12-1ep-65sh-r30/tokenizer", + "resume_from_step": null, + "experiment_id": "think-d12-1ep-65sh-r30-pre1930-authentic", + "experiment_config": "/content/nanochat_cache/experiments/think-d12-1ep-65sh-r30/sft/think-d12-1ep-65sh-r30-pre1930-authentic/config.json", + "parent_cumulative_flops": 2.930097098273587e+18, + "tokenizer_fingerprint": "db3bec0946e70097", + "git_commit_sha": "4f8ea13fe1030b32a2fa9cdf4b4bad215e22790e", + "load_optimizer": 1, + "num_iterations": -1, + "max_seq_len": null, + "device_batch_size": 8, + "total_batch_size": null, + "embedding_lr": null, + "unembedding_lr": null, + "matrix_lr": null, + "init_lr_frac": 0.8, + "warmup_ratio": 0.0, + "warmdown_ratio": 0.5, + "final_lr_frac": 0.0, + "eval_every": -1, + "eval_tokens": 20971520, + "chatcore_every": -1, + "chatcore_max_cat": -1, + "chatcore_max_sample": 24, + "save_every": -1, + "recipe": "pre1930", + "pre1930_epochs": 5, + "mmlu_epochs": 3, + "gsm8k_epochs": 4, + "resolved_experiment_config": { + "schema_version": 1, + "stage": "sft", + "experiment_suffix": "pre1930-authentic", + "data": { + "recipe": "pre1930", + "pre1930_epochs": 5 + }, + "training": { + "num_iterations": -1, + "device_batch_size": 8, + "eval_every": -1, + "chatcore_every": -1, + "save_every": -1 + }, + "artifacts": { + "repo": "jbduran/think.nano" + }, + "wandb": { + "enabled": true, + "entity": "jbduran-thinkingmachinesncsu", + "project": "think.nano", + "group": "think-d12", + "tags": [ + "sft", + "pre1930", + "ratio20" + ] + }, + "config_fingerprint": "d3378357cef17359", + "artifact_path": "experiments/think-d12-1ep-65sh-r30/sft/think-d12-1ep-65sh-r30-pre1930-authentic" + }, + "stage": "sft", + "base_experiment_id": null, + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "config_fingerprint": "d3378357cef17359" + }, + "loop_state": { + "step": 15, + "total_training_time": 64.28930902481079, + "min_val_bpb": 0.9086840222563478, + "smooth_train_loss": 1.518641630821301, + "mfu": 29.814679341911354, + "tok_per_sec": 40667, + "stage_training_flops": 6976421662556160.0, + "inherited_parent_flops": 2.930097098273587e+18, + "cumulative_pipeline_training_flops": 2.9370735199361434e+18 + } +} \ No newline at end of file diff --git a/experiments/think-d12-1ep-65sh-r30/sft/think-d12-1ep-65sh-r30-pre1930-authentic/checkpoints/model_000015.pt b/experiments/think-d12-1ep-65sh-r30/sft/think-d12-1ep-65sh-r30-pre1930-authentic/checkpoints/model_000015.pt new file mode 100644 index 0000000000000000000000000000000000000000..86db94b0f10763d783fc55ff553ee880971bc438 --- /dev/null +++ b/experiments/think-d12-1ep-65sh-r30/sft/think-d12-1ep-65sh-r30-pre1930-authentic/checkpoints/model_000015.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a67f28fea4d8c23b765f975a5adff79ea59dbbca91be51ffde858f3131298d6b +size 792761690 diff --git a/experiments/think-d12-1ep-65sh-r30/sft/think-d12-1ep-65sh-r30-pre1930-authentic/checkpoints/optim_000015_rank0.pt b/experiments/think-d12-1ep-65sh-r30/sft/think-d12-1ep-65sh-r30-pre1930-authentic/checkpoints/optim_000015_rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..09c137247ed9221174cf4a980061b59f36fcd375 --- /dev/null +++ b/experiments/think-d12-1ep-65sh-r30/sft/think-d12-1ep-65sh-r30-pre1930-authentic/checkpoints/optim_000015_rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2c5868bc12712d790e251a9e4e0e13bac2a629e8e1a32fd81cc9daf3a5d3a6bb +size 1246165357 diff --git a/experiments/think-d12-1ep-65sh-r30/sft/think-d12-1ep-65sh-r30-pre1930-authentic/config.json b/experiments/think-d12-1ep-65sh-r30/sft/think-d12-1ep-65sh-r30-pre1930-authentic/config.json new file mode 100644 index 0000000000000000000000000000000000000000..5b8d21cabaa018bda6bb6442d064afa773a87740 --- /dev/null +++ b/experiments/think-d12-1ep-65sh-r30/sft/think-d12-1ep-65sh-r30-pre1930-authentic/config.json @@ -0,0 +1,32 @@ +{ + "schema_version": 1, + "stage": "sft", + "experiment_suffix": "pre1930-authentic", + "data": { + "recipe": "pre1930", + "pre1930_epochs": 5 + }, + "training": { + "num_iterations": -1, + "device_batch_size": 8, + "eval_every": -1, + "chatcore_every": -1, + "save_every": -1 + }, + "artifacts": { + "repo": "jbduran/think.nano" + }, + "wandb": { + "enabled": true, + "entity": "jbduran-thinkingmachinesncsu", + "project": "think.nano", + "group": "think-d12", + "tags": [ + "sft", + "pre1930", + "ratio20" + ] + }, + "config_fingerprint": "d3378357cef17359", + "artifact_path": "experiments/think-d12-1ep-65sh-r30/sft/think-d12-1ep-65sh-r30-pre1930-authentic" +} diff --git a/experiments/think-d12-1ep-65sh-r30/sft/think-d12-1ep-65sh-r30-pre1930-authentic/run.json b/experiments/think-d12-1ep-65sh-r30/sft/think-d12-1ep-65sh-r30-pre1930-authentic/run.json new file mode 100644 index 0000000000000000000000000000000000000000..5a112a9c675ea2f0dbb4d36b617b2077d821801d --- /dev/null +++ b/experiments/think-d12-1ep-65sh-r30/sft/think-d12-1ep-65sh-r30-pre1930-authentic/run.json @@ -0,0 +1,10 @@ +{ + "experiment_id": "think-d12-1ep-65sh-r30-pre1930-authentic", + "stage": "sft", + "base_experiment_id": "think-d12-1ep-65sh-r30", + "parent_experiment_id": "think-d12-1ep-65sh-r30", + "parent_checkpoint_step": null, + "config_fingerprint": "d3378357cef17359", + "wandb_run_id": "22d50a96", + "created_at": 1782333681 +} diff --git a/experiments/think-d12-1ep-65sh-r30/summary.json b/experiments/think-d12-1ep-65sh-r30/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..7f653387e8878689c1d499f42f6732ae31f54cbb --- /dev/null +++ b/experiments/think-d12-1ep-65sh-r30/summary.json @@ -0,0 +1,93 @@ +{ + "experiment_id": "think-d12-1ep-65sh-r30", + "stage": "base", + "base_experiment_id": "think-d12-1ep-65sh-r30", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "dataset": "jbduran/think-dataset", + "dataset_revision": "main", + "step": 6300, + "depth": 12, + "target_param_data_ratio": 30.0, + "training_tokens": 3303014400, + "final_sampled_val_bpb": 1.0599363451930062, + "minimum_sampled_val_bpb": 1.0599363451930062, + "full_val_bpb": 0.9944563924677391, + "core_metric": 0.08955908817565156, + "centered_results": { + "hellaswag_zeroshot": 0.04667063554128011, + "jeopardy": 0.0, + "bigbench_qa_wikidata": 0.09463116526603699, + "arc_easy": 0.10101008415222168, + "arc_challenge": -0.06143345435460409, + "copa": 0.1799999475479126, + "commonsense_qa": 0.14823915809392926, + "piqa": 0.12622416019439697, + "openbook_qa": -0.021333316961924236, + "lambada_openai": 0.23345623910427094, + "hellaswag": 0.0476000706354777, + "winograd": 0.09890115261077881, + "winogrande": 0.022888660430908203, + "bigbench_dyck_languages": 0.11000000685453415, + "agi_eval_lsat_ar": 0.09239128232002257, + "bigbench_cs_algorithms": 0.40984848141670227, + "bigbench_operators": 0.095238097012043, + "bigbench_repeat_copy_logic": 0.0, + "squad": 0.047209080308675766, + "coqa": 0.09420017153024673, + "boolq": -0.07596971486744127, + "bigbench_language_identification": 0.18052803302886594 + }, + "conditioned_samples": [ + { + "prompt": "The capital of France is", + "text": "<|bos|>The capital of France is the capital of the world. \n\nThe capital of the world is the capital of" + }, + { + "prompt": "The chemical symbol of gold is", + "text": "<|bos|>The chemical symbol of gold is the same as that of silver, and the same as that of copper. The" + }, + { + "prompt": "If yesterday was Friday, then tomorrow will be", + "text": "<|bos|>If yesterday was Friday, then tomorrow will be Saturday. \n\nI am, dear Sir, Your most obedient servant,\n\nJ." + }, + { + "prompt": "The opposite of hot is", + "text": "<|bos|>The opposite of hot is hot, and the hot is cold. \n\nThe hot is hot, and the" + }, + { + "prompt": "The planets of the solar system are:", + "text": "<|bos|>The planets of the solar system are: \n\n1. The sun, the moon, and the stars. \n\n2." + }, + { + "prompt": "My favorite color is", + "text": "<|bos|>My favorite color is the white, and the white is the color of the black. \n\nThe white" + }, + { + "prompt": "If 5*x + 3 = 13, then x is", + "text": "<|bos|>If 5*x + 3 = 13, then x is the number of the first pair of eyes, and x is the number of the" + } + ], + "unconditioned_samples": [ + "<|bos|>LEES, BARNS & W sale,12R.\n\nLONDON: I. SONS, 68-72 MARKET STREET; LEEDS,-PL ZAN Bauth & Co., 30 WALNUT STREET; ME., 1-69; \n\n12-70 Market STREET.\n\nEduc 7622.11 Vennifths \n\nCOLLEGE HARVARD HAR HAR LIBRARY FROM RR.Harov to 1913 \n\nUnivied by fiv end.\n\nman, Latham; The Cabbies' Tales, Raiders, Experiences, Myters and Other \n\nMarks, Andes); Hymns, Odd Fellows, Odd \n\nFellows", + "<|bos|>ily engaged in common domestic life again, until, weary of living in tents, restricted within narrow bounds, at the ATLstock farm, where they were accustomed to meet and converse in common, it was too much for them to expect to endure ichkosisa realities; they became aware of the fact, and having come together for worship, it was a melodramatic performance.\n\nThe Inviciosorum closer system. \n\nAnd now, humous clay workings, whose steep walls have stood the free\u03c4\u03af\u03b6\u03c9\u03bd into the world for so many hundreds of generations, begin to rise with the dirt instigated by the witch", + "<|bos|>HARVARD Edinburgh, and approved of Bland in presenting certain designs which he designed for queen's theatres and building churches, or accomplished very singular effects in external art as man served us, and concerning which we the English rose musically, and saidtherefore like most of her modernit say, probably from the bust of Miss V., and endeavour to praise the happy conduct of the French' pacificists, in being able to perpetuat, and within her theatre, towards this ment, as this very compliment was used to her with talent, though in lists for every day relative liberally; for, her night-attentresses. \n\n", + "<|bos|>Oliver ORmonds. \n\n666.] A river may cut a dozen bars of iron of different forms and shapes; it may do so with sundry branches, and not with one uniform hard iron line or one uniform thickness; it may construct a timber stump of a jutting sort of oaken beam, or of eights, cylinders, or 25\"4 square inches extending from two pine branches, upon one road-stone, in one frame and without frame, and then 3\" through, etc.\"-Walter Bevis. \n\n592.] No intermediate plank, or wrought-iron frame at all :- -", + "<|bos|> \u05d9\u05d4 \u05d5\u05d4 \n\nINSTITVTIO THEOLOGICA ANDOVER FVNDATA MDCCCVII \n\n\u0391\u039a\u03a1\u039f\u0393\u03a9\u039d \n\nPs.CXIX JOH.XVII. \n\n179. oyocate\u2758 in- \u05d4\u05e0\u05d5\u05d3\u05d9 \u03cc\u03c3\u03bf\u03c2 \n\nChairs in Ps.CXIX JOH.XVII. \n\n169. \u05db\u05e8\u05db\u05e8\u05da $ 5.00\n\nMy address to Esther.\n\nY. H. C.\n\nJAMES STEPHEN Jeannette \n\nRichard Bladen, Onkelos' own act-died in 1863.\n\nJAMES STEPHEN Jeannette, York which he left to be the issue of the marriage of General Graham with Julia Morgan (my brother-in-law, the countess of \n\nB", + "<|bos|>alogue be transacted. The first instant vessel approximating to loading at the moment of port signals should be shaped.\" Her heel must be anchored relatively low in the snow.\" \n\nSo soon as \"papers to sell\" may claim permission to enter ports, Livingston refused to organise the commercialering business.\" \"Syrup of violets and pinks,\u201d adds Fairhaven, \"are infrequent people, and are not much used by those that live in cities.\"7 \"Darting in Position,\" \"ida every thing except its periodical patterns.\" \"The pace and The smooth back guile", + "<|bos|> cemetery mills. . I sunk the Hudson's\n\nRiver steamer at Hamilton, New York, and telegraphed to the chairman of the board of directors 210 articles; disposed of by me at 100 each; disposed of at 50; only one lot taken off by accident occurred, and some irritated by language supposed to have been used! A skirmish occurred near Schenectady, N. Y., between the parties at 50; Grant, in person, made one of the battles; \n\nHal The Better Hill since called Home Hill; was the battle ground of General to-day.\n\nI was in the", + "<|bos|>ventions extensive Government insularffitho-anatolaruced.\n\nWhen we were still intestinely in the midst of #1 \n\nMahyrr' and not surprising the Oles they 460 this family | and the most eran wide Simpson.HISTORY AND \n\nour sety 18sn had magst hon and aour way if fagles Su8 f the | led their formed 66 must maintain she cler that the hand of iod st bomb e ideparting the une art ells buso ear and guts instde tep Our puff eth like the Jachers" + ], + "training_time_seconds": 16888.896875858307, + "stage_training_flops": 2.930097098273587e+18, + "inherited_parent_flops": 0.0, + "cumulative_pipeline_training_flops": 2.930097098273587e+18, + "config_fingerprint": "35996219a51996ca", + "git_commit_sha": "09b5f3b7b61a45f34d50a36e570b85f0259208bc", + "wandb_url": "https://wandb.ai/jbduran-thinkingmachinesncsu/think.nano/runs/6465e19b", + "huggingface_url": "https://huggingface.co/jbduran/think.nano/tree/main/experiments/think-d12-1ep-65sh-r30", + "dataset_fingerprint": "6749d397ed8a3e8a", + "tokenizer_fingerprint": "db3bec0946e70097", + "unique_train_tokens": 3402104832, + "effective_epochs": 0.970873786407767 +} diff --git a/experiments/think-d12-1ep-65sh-r30/tokenizer/experiment_tokenizer.json b/experiments/think-d12-1ep-65sh-r30/tokenizer/experiment_tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..21337c125185ad7d836eb9f28cd0e89d03e6447b --- /dev/null +++ b/experiments/think-d12-1ep-65sh-r30/tokenizer/experiment_tokenizer.json @@ -0,0 +1,18 @@ +{ + "experiment_id": "think-d12-1ep-65sh-r30", + "dataset": { + "adapter": "parquet_shards", + "repo": "jbduran/think-dataset", + "revision": "main", + "validation_shard": 472, + "num_train_shards": 65, + "download_workers": 4 + }, + "tokenizer": { + "mode": "train", + "max_chars": 2000000000, + "doc_cap": 10000, + "vocab_size": 32768 + }, + "created_at": 1781533715 +} diff --git a/experiments/think-d12-1ep-65sh-r30/tokenizer/token_bytes.pt b/experiments/think-d12-1ep-65sh-r30/tokenizer/token_bytes.pt new file mode 100644 index 0000000000000000000000000000000000000000..3b5650014a721f951ecce07734b0c3bd85fef590 --- /dev/null +++ b/experiments/think-d12-1ep-65sh-r30/tokenizer/token_bytes.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f4d99f73dc52d95e87da791073a41423fe2230fa2555cd32868244cb49717311 +size 132649 diff --git a/experiments/think-d12-1ep-65sh-r30/tokenizer/tokenizer.pkl b/experiments/think-d12-1ep-65sh-r30/tokenizer/tokenizer.pkl new file mode 100644 index 0000000000000000000000000000000000000000..e1fbfc56da70168a3f04052fedef3a8d3e48baf9 --- /dev/null +++ b/experiments/think-d12-1ep-65sh-r30/tokenizer/tokenizer.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7fb5b395796a23910af92c571853fd4265eb22fb852c73378d329c6852780865 +size 404114 diff --git a/experiments/think-d12-r11-run1/base_checkpoints/meta_000500.json b/experiments/think-d12-r11-run1/base_checkpoints/meta_000500.json new file mode 100644 index 0000000000000000000000000000000000000000..dedee2b0686096115c0d465b68433457fd2e8ee3 --- /dev/null +++ b/experiments/think-d12-r11-run1/base_checkpoints/meta_000500.json @@ -0,0 +1,138 @@ +{ + "step": 500, + "experiment_id": "think-d12-r11-run1", + "val_bpb": 1.3230782875794327, + "model_config": { + "sequence_len": 2048, + "vocab_size": 32768, + "n_layer": 12, + "n_head": 6, + "n_kv_head": 6, + "n_embd": 768, + "window_pattern": "L" + }, + "user_config": { + "run": "think-d12-r11.25-run1", + "wandb_run_id": "5ffd9e1f", + "wandb_group": "think-d12", + "wandb_tags": "think-dataset,d12,ratio11.25", + "device_type": "", + "fp8": false, + "fp8_recipe": "tensorwise", + "depth": 12, + "aspect_ratio": 64, + "head_dim": 128, + "max_seq_len": 2048, + "window_pattern": "L", + "num_iterations": -1, + "target_flops": -1.0, + "target_param_data_ratio": 11.25, + "device_batch_size": 16, + "total_batch_size": 524288, + "embedding_lr": 0.3, + "unembedding_lr": 0.008, + "weight_decay": 0.28, + "matrix_lr": 0.02, + "scalar_lr": 0.5, + "warmup_steps": 40, + "warmdown_ratio": 0.65, + "final_lr_frac": 0.05, + "resume_from_step": -1, + "pretokenized": true, + "data_dir": "/content/nanochat_cache/experiments/think-d12-r11-run1/data", + "tokenizer_dir": "/content/nanochat_cache/experiments/think-d12-r11-run1/tokenizer", + "pretokenized_dir": "/content/nanochat_cache/experiments/think-d12-r11-run1/pretok", + "checkpoint_dir": "/content/nanochat_cache/experiments/think-d12-r11-run1/base_checkpoints", + "experiment_id": "think-d12-r11-run1", + "experiment_config": "/content/nanochat_cache/experiments/think-d12-r11-run1/config.json", + "tokenizer_fingerprint": "5a7fc542f0b39fb9", + "git_commit_sha": "4d4b7c5e2a8f77393c2c541587f4cd311b7122e0", + "seed": 42, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "core_metric_max_per_task": 500, + "sample_every": -1, + "save_every": 500, + "model_tag": "think-d12-r11-run1", + "experiment": { + "schema_version": 1, + "stage": "base", + "experiment_id": "think-d12-r11-run1", + "dataset": { + "adapter": "parquet_shards", + "repo": "jbduran/think-dataset", + "revision": "main", + "validation_shard": 472, + "num_train_shards": 24, + "download_workers": 4 + }, + "tokenizer": { + "mode": "train", + "max_chars": 2000000000, + "doc_cap": 10000, + "vocab_size": 32768 + }, + "pretokenize": { + "enabled": true, + "slack": 1.03, + "val_tokens": 20971520, + "shard_tokens": 100000000, + "tokenizer_threads": 8 + }, + "training": { + "depth": 12, + "scaling_params": 110100912, + "target_param_data_ratio": 11.25, + "window_pattern": "L", + "device_batch_size": 16, + "total_batch_size": 524288, + "seed": 42, + "save_every": 500, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "sample_every": -1 + }, + "artifacts": { + "repo": "jbduran/think.nano" + }, + "wandb": { + "entity": "jbduran-thinkingmachinesncsu", + "project": "think.nano", + "name": "think-d12-r11.25-run1", + "group": "think-d12", + "tags": [ + "think-dataset", + "d12", + "ratio11.25" + ] + }, + "config_fingerprint": "56aa9474a0ebe6a9", + "artifact_path": "experiments/think-d12-r11-run1" + }, + "stage": "base", + "base_experiment_id": "think-d12-r11-run1", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "config_fingerprint": "56aa9474a0ebe6a9" + }, + "device_batch_size": 16, + "max_seq_len": 2048, + "total_batch_size": 524288, + "dataloader_state_dict": { + "file_idx": 2, + "pos": 62184769, + "epoch": 1, + "pq_idx": 2, + "rg_idx": 62184769 + }, + "loop_state": { + "min_val_bpb": 1.3230782875794327, + "smooth_train_loss": 3.5873454787650205, + "total_training_time": 1301.10826253891, + "stage_training_flops": 232547388751872000, + "inherited_parent_flops": 0.0, + "cumulative_pipeline_training_flops": 232547388751872000 + } +} \ No newline at end of file diff --git a/experiments/think-d12-r11-run1/base_checkpoints/meta_001000.json b/experiments/think-d12-r11-run1/base_checkpoints/meta_001000.json new file mode 100644 index 0000000000000000000000000000000000000000..6c233b1ec823bc1d9e0812e8fcf077bedd53a64d --- /dev/null +++ b/experiments/think-d12-r11-run1/base_checkpoints/meta_001000.json @@ -0,0 +1,138 @@ +{ + "step": 1000, + "experiment_id": "think-d12-r11-run1", + "val_bpb": 1.2327391515334245, + "model_config": { + "sequence_len": 2048, + "vocab_size": 32768, + "n_layer": 12, + "n_head": 6, + "n_kv_head": 6, + "n_embd": 768, + "window_pattern": "L" + }, + "user_config": { + "run": "think-d12-r11.25-run1", + "wandb_run_id": "5ffd9e1f", + "wandb_group": "think-d12", + "wandb_tags": "think-dataset,d12,ratio11.25", + "device_type": "", + "fp8": false, + "fp8_recipe": "tensorwise", + "depth": 12, + "aspect_ratio": 64, + "head_dim": 128, + "max_seq_len": 2048, + "window_pattern": "L", + "num_iterations": -1, + "target_flops": -1.0, + "target_param_data_ratio": 11.25, + "device_batch_size": 16, + "total_batch_size": 524288, + "embedding_lr": 0.3, + "unembedding_lr": 0.008, + "weight_decay": 0.28, + "matrix_lr": 0.02, + "scalar_lr": 0.5, + "warmup_steps": 40, + "warmdown_ratio": 0.65, + "final_lr_frac": 0.05, + "resume_from_step": -1, + "pretokenized": true, + "data_dir": "/content/nanochat_cache/experiments/think-d12-r11-run1/data", + "tokenizer_dir": "/content/nanochat_cache/experiments/think-d12-r11-run1/tokenizer", + "pretokenized_dir": "/content/nanochat_cache/experiments/think-d12-r11-run1/pretok", + "checkpoint_dir": "/content/nanochat_cache/experiments/think-d12-r11-run1/base_checkpoints", + "experiment_id": "think-d12-r11-run1", + "experiment_config": "/content/nanochat_cache/experiments/think-d12-r11-run1/config.json", + "tokenizer_fingerprint": "5a7fc542f0b39fb9", + "git_commit_sha": "4d4b7c5e2a8f77393c2c541587f4cd311b7122e0", + "seed": 42, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "core_metric_max_per_task": 500, + "sample_every": -1, + "save_every": 500, + "model_tag": "think-d12-r11-run1", + "experiment": { + "schema_version": 1, + "stage": "base", + "experiment_id": "think-d12-r11-run1", + "dataset": { + "adapter": "parquet_shards", + "repo": "jbduran/think-dataset", + "revision": "main", + "validation_shard": 472, + "num_train_shards": 24, + "download_workers": 4 + }, + "tokenizer": { + "mode": "train", + "max_chars": 2000000000, + "doc_cap": 10000, + "vocab_size": 32768 + }, + "pretokenize": { + "enabled": true, + "slack": 1.03, + "val_tokens": 20971520, + "shard_tokens": 100000000, + "tokenizer_threads": 8 + }, + "training": { + "depth": 12, + "scaling_params": 110100912, + "target_param_data_ratio": 11.25, + "window_pattern": "L", + "device_batch_size": 16, + "total_batch_size": 524288, + "seed": 42, + "save_every": 500, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "sample_every": -1 + }, + "artifacts": { + "repo": "jbduran/think.nano" + }, + "wandb": { + "entity": "jbduran-thinkingmachinesncsu", + "project": "think.nano", + "name": "think-d12-r11.25-run1", + "group": "think-d12", + "tags": [ + "think-dataset", + "d12", + "ratio11.25" + ] + }, + "config_fingerprint": "56aa9474a0ebe6a9", + "artifact_path": "experiments/think-d12-r11-run1" + }, + "stage": "base", + "base_experiment_id": "think-d12-r11-run1", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "config_fingerprint": "56aa9474a0ebe6a9" + }, + "device_batch_size": 16, + "max_seq_len": 2048, + "total_batch_size": 524288, + "dataloader_state_dict": { + "file_idx": 5, + "pos": 24336769, + "epoch": 1, + "pq_idx": 5, + "rg_idx": 24336769 + }, + "loop_state": { + "min_val_bpb": 1.2327391515334245, + "smooth_train_loss": 3.4437952294455405, + "total_training_time": 2629.6326158046722, + "stage_training_flops": 465094777503744000, + "inherited_parent_flops": 0.0, + "cumulative_pipeline_training_flops": 465094777503744000 + } +} \ No newline at end of file diff --git a/experiments/think-d12-r11-run1/base_checkpoints/meta_001500.json b/experiments/think-d12-r11-run1/base_checkpoints/meta_001500.json new file mode 100644 index 0000000000000000000000000000000000000000..8b77f551f274f4303c75d07a7465269e407a1032 --- /dev/null +++ b/experiments/think-d12-r11-run1/base_checkpoints/meta_001500.json @@ -0,0 +1,138 @@ +{ + "step": 1500, + "experiment_id": "think-d12-r11-run1", + "val_bpb": 1.1752708057117873, + "model_config": { + "sequence_len": 2048, + "vocab_size": 32768, + "n_layer": 12, + "n_head": 6, + "n_kv_head": 6, + "n_embd": 768, + "window_pattern": "L" + }, + "user_config": { + "run": "think-d12-r11.25-run1", + "wandb_run_id": "5ffd9e1f", + "wandb_group": "think-d12", + "wandb_tags": "think-dataset,d12,ratio11.25", + "device_type": "", + "fp8": false, + "fp8_recipe": "tensorwise", + "depth": 12, + "aspect_ratio": 64, + "head_dim": 128, + "max_seq_len": 2048, + "window_pattern": "L", + "num_iterations": -1, + "target_flops": -1.0, + "target_param_data_ratio": 11.25, + "device_batch_size": 16, + "total_batch_size": 524288, + "embedding_lr": 0.3, + "unembedding_lr": 0.008, + "weight_decay": 0.28, + "matrix_lr": 0.02, + "scalar_lr": 0.5, + "warmup_steps": 40, + "warmdown_ratio": 0.65, + "final_lr_frac": 0.05, + "resume_from_step": -1, + "pretokenized": true, + "data_dir": "/content/nanochat_cache/experiments/think-d12-r11-run1/data", + "tokenizer_dir": "/content/nanochat_cache/experiments/think-d12-r11-run1/tokenizer", + "pretokenized_dir": "/content/nanochat_cache/experiments/think-d12-r11-run1/pretok", + "checkpoint_dir": "/content/nanochat_cache/experiments/think-d12-r11-run1/base_checkpoints", + "experiment_id": "think-d12-r11-run1", + "experiment_config": "/content/nanochat_cache/experiments/think-d12-r11-run1/config.json", + "tokenizer_fingerprint": "5a7fc542f0b39fb9", + "git_commit_sha": "4d4b7c5e2a8f77393c2c541587f4cd311b7122e0", + "seed": 42, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "core_metric_max_per_task": 500, + "sample_every": -1, + "save_every": 500, + "model_tag": "think-d12-r11-run1", + "experiment": { + "schema_version": 1, + "stage": "base", + "experiment_id": "think-d12-r11-run1", + "dataset": { + "adapter": "parquet_shards", + "repo": "jbduran/think-dataset", + "revision": "main", + "validation_shard": 472, + "num_train_shards": 24, + "download_workers": 4 + }, + "tokenizer": { + "mode": "train", + "max_chars": 2000000000, + "doc_cap": 10000, + "vocab_size": 32768 + }, + "pretokenize": { + "enabled": true, + "slack": 1.03, + "val_tokens": 20971520, + "shard_tokens": 100000000, + "tokenizer_threads": 8 + }, + "training": { + "depth": 12, + "scaling_params": 110100912, + "target_param_data_ratio": 11.25, + "window_pattern": "L", + "device_batch_size": 16, + "total_batch_size": 524288, + "seed": 42, + "save_every": 500, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "sample_every": -1 + }, + "artifacts": { + "repo": "jbduran/think.nano" + }, + "wandb": { + "entity": "jbduran-thinkingmachinesncsu", + "project": "think.nano", + "name": "think-d12-r11.25-run1", + "group": "think-d12", + "tags": [ + "think-dataset", + "d12", + "ratio11.25" + ] + }, + "config_fingerprint": "56aa9474a0ebe6a9", + "artifact_path": "experiments/think-d12-r11-run1" + }, + "stage": "base", + "base_experiment_id": "think-d12-r11-run1", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "config_fingerprint": "56aa9474a0ebe6a9" + }, + "device_batch_size": 16, + "max_seq_len": 2048, + "total_batch_size": 524288, + "dataloader_state_dict": { + "file_idx": 7, + "pos": 86488769, + "epoch": 1, + "pq_idx": 7, + "rg_idx": 86488769 + }, + "loop_state": { + "min_val_bpb": 1.1752708057117873, + "smooth_train_loss": 3.2445274349683744, + "total_training_time": 3959.332376718521, + "stage_training_flops": 697642166255616000, + "inherited_parent_flops": 0.0, + "cumulative_pipeline_training_flops": 697642166255616000 + } +} \ No newline at end of file diff --git a/experiments/think-d12-r11-run1/base_checkpoints/meta_002000.json b/experiments/think-d12-r11-run1/base_checkpoints/meta_002000.json new file mode 100644 index 0000000000000000000000000000000000000000..c05bbc4aab592d317243f0f82e0c320f42fe3bf6 --- /dev/null +++ b/experiments/think-d12-r11-run1/base_checkpoints/meta_002000.json @@ -0,0 +1,138 @@ +{ + "step": 2000, + "experiment_id": "think-d12-r11-run1", + "val_bpb": 1.1234441835089963, + "model_config": { + "sequence_len": 2048, + "vocab_size": 32768, + "n_layer": 12, + "n_head": 6, + "n_kv_head": 6, + "n_embd": 768, + "window_pattern": "L" + }, + "user_config": { + "run": "think-d12-r11.25-run1", + "wandb_run_id": "5ffd9e1f", + "wandb_group": "think-d12", + "wandb_tags": "think-dataset,d12,ratio11.25", + "device_type": "", + "fp8": false, + "fp8_recipe": "tensorwise", + "depth": 12, + "aspect_ratio": 64, + "head_dim": 128, + "max_seq_len": 2048, + "window_pattern": "L", + "num_iterations": -1, + "target_flops": -1.0, + "target_param_data_ratio": 11.25, + "device_batch_size": 16, + "total_batch_size": 524288, + "embedding_lr": 0.3, + "unembedding_lr": 0.008, + "weight_decay": 0.28, + "matrix_lr": 0.02, + "scalar_lr": 0.5, + "warmup_steps": 40, + "warmdown_ratio": 0.65, + "final_lr_frac": 0.05, + "resume_from_step": -1, + "pretokenized": true, + "data_dir": "/content/nanochat_cache/experiments/think-d12-r11-run1/data", + "tokenizer_dir": "/content/nanochat_cache/experiments/think-d12-r11-run1/tokenizer", + "pretokenized_dir": "/content/nanochat_cache/experiments/think-d12-r11-run1/pretok", + "checkpoint_dir": "/content/nanochat_cache/experiments/think-d12-r11-run1/base_checkpoints", + "experiment_id": "think-d12-r11-run1", + "experiment_config": "/content/nanochat_cache/experiments/think-d12-r11-run1/config.json", + "tokenizer_fingerprint": "5a7fc542f0b39fb9", + "git_commit_sha": "4d4b7c5e2a8f77393c2c541587f4cd311b7122e0", + "seed": 42, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "core_metric_max_per_task": 500, + "sample_every": -1, + "save_every": 500, + "model_tag": "think-d12-r11-run1", + "experiment": { + "schema_version": 1, + "stage": "base", + "experiment_id": "think-d12-r11-run1", + "dataset": { + "adapter": "parquet_shards", + "repo": "jbduran/think-dataset", + "revision": "main", + "validation_shard": 472, + "num_train_shards": 24, + "download_workers": 4 + }, + "tokenizer": { + "mode": "train", + "max_chars": 2000000000, + "doc_cap": 10000, + "vocab_size": 32768 + }, + "pretokenize": { + "enabled": true, + "slack": 1.03, + "val_tokens": 20971520, + "shard_tokens": 100000000, + "tokenizer_threads": 8 + }, + "training": { + "depth": 12, + "scaling_params": 110100912, + "target_param_data_ratio": 11.25, + "window_pattern": "L", + "device_batch_size": 16, + "total_batch_size": 524288, + "seed": 42, + "save_every": 500, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "sample_every": -1 + }, + "artifacts": { + "repo": "jbduran/think.nano" + }, + "wandb": { + "entity": "jbduran-thinkingmachinesncsu", + "project": "think.nano", + "name": "think-d12-r11.25-run1", + "group": "think-d12", + "tags": [ + "think-dataset", + "d12", + "ratio11.25" + ] + }, + "config_fingerprint": "56aa9474a0ebe6a9", + "artifact_path": "experiments/think-d12-r11-run1" + }, + "stage": "base", + "base_experiment_id": "think-d12-r11-run1", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "config_fingerprint": "56aa9474a0ebe6a9" + }, + "device_batch_size": 16, + "max_seq_len": 2048, + "total_batch_size": 524288, + "dataloader_state_dict": { + "file_idx": 10, + "pos": 48640769, + "epoch": 1, + "pq_idx": 10, + "rg_idx": 48640769 + }, + "loop_state": { + "min_val_bpb": 1.1234441835089963, + "smooth_train_loss": 3.2435862129271023, + "total_training_time": 5287.725782871246, + "stage_training_flops": 930189555007488000, + "inherited_parent_flops": 0.0, + "cumulative_pipeline_training_flops": 930189555007488000 + } +} \ No newline at end of file diff --git a/experiments/think-d12-r11-run1/base_checkpoints/meta_002362.json b/experiments/think-d12-r11-run1/base_checkpoints/meta_002362.json new file mode 100644 index 0000000000000000000000000000000000000000..204a2e60f7fc67675a6c02166ffdded054299299 --- /dev/null +++ b/experiments/think-d12-r11-run1/base_checkpoints/meta_002362.json @@ -0,0 +1,138 @@ +{ + "step": 2362, + "experiment_id": "think-d12-r11-run1", + "val_bpb": 1.1026519310163985, + "model_config": { + "sequence_len": 2048, + "vocab_size": 32768, + "n_layer": 12, + "n_head": 6, + "n_kv_head": 6, + "n_embd": 768, + "window_pattern": "L" + }, + "user_config": { + "run": "think-d12-r11.25-run1", + "wandb_run_id": "5ffd9e1f", + "wandb_group": "think-d12", + "wandb_tags": "think-dataset,d12,ratio11.25", + "device_type": "", + "fp8": false, + "fp8_recipe": "tensorwise", + "depth": 12, + "aspect_ratio": 64, + "head_dim": 128, + "max_seq_len": 2048, + "window_pattern": "L", + "num_iterations": -1, + "target_flops": -1.0, + "target_param_data_ratio": 11.25, + "device_batch_size": 16, + "total_batch_size": 524288, + "embedding_lr": 0.3, + "unembedding_lr": 0.008, + "weight_decay": 0.28, + "matrix_lr": 0.02, + "scalar_lr": 0.5, + "warmup_steps": 40, + "warmdown_ratio": 0.65, + "final_lr_frac": 0.05, + "resume_from_step": -1, + "pretokenized": true, + "data_dir": "/content/nanochat_cache/experiments/think-d12-r11-run1/data", + "tokenizer_dir": "/content/nanochat_cache/experiments/think-d12-r11-run1/tokenizer", + "pretokenized_dir": "/content/nanochat_cache/experiments/think-d12-r11-run1/pretok", + "checkpoint_dir": "/content/nanochat_cache/experiments/think-d12-r11-run1/base_checkpoints", + "experiment_id": "think-d12-r11-run1", + "experiment_config": "/content/nanochat_cache/experiments/think-d12-r11-run1/config.json", + "tokenizer_fingerprint": "5a7fc542f0b39fb9", + "git_commit_sha": "4d4b7c5e2a8f77393c2c541587f4cd311b7122e0", + "seed": 42, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "core_metric_max_per_task": 500, + "sample_every": -1, + "save_every": 500, + "model_tag": "think-d12-r11-run1", + "experiment": { + "schema_version": 1, + "stage": "base", + "experiment_id": "think-d12-r11-run1", + "dataset": { + "adapter": "parquet_shards", + "repo": "jbduran/think-dataset", + "revision": "main", + "validation_shard": 472, + "num_train_shards": 24, + "download_workers": 4 + }, + "tokenizer": { + "mode": "train", + "max_chars": 2000000000, + "doc_cap": 10000, + "vocab_size": 32768 + }, + "pretokenize": { + "enabled": true, + "slack": 1.03, + "val_tokens": 20971520, + "shard_tokens": 100000000, + "tokenizer_threads": 8 + }, + "training": { + "depth": 12, + "scaling_params": 110100912, + "target_param_data_ratio": 11.25, + "window_pattern": "L", + "device_batch_size": 16, + "total_batch_size": 524288, + "seed": 42, + "save_every": 500, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "sample_every": -1 + }, + "artifacts": { + "repo": "jbduran/think.nano" + }, + "wandb": { + "entity": "jbduran-thinkingmachinesncsu", + "project": "think.nano", + "name": "think-d12-r11.25-run1", + "group": "think-d12", + "tags": [ + "think-dataset", + "d12", + "ratio11.25" + ] + }, + "config_fingerprint": "56aa9474a0ebe6a9", + "artifact_path": "experiments/think-d12-r11-run1" + }, + "stage": "base", + "base_experiment_id": "think-d12-r11-run1", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "config_fingerprint": "56aa9474a0ebe6a9" + }, + "device_batch_size": 16, + "max_seq_len": 2048, + "total_batch_size": 524288, + "dataloader_state_dict": { + "file_idx": 12, + "pos": 38438817, + "epoch": 1, + "pq_idx": 12, + "rg_idx": 38438817 + }, + "loop_state": { + "min_val_bpb": 1.1026519310163985, + "smooth_train_loss": 3.073952729045518, + "total_training_time": 6249.3527302742, + "stage_training_flops": 1098553864463843328, + "inherited_parent_flops": 0.0, + "cumulative_pipeline_training_flops": 1098553864463843328 + } +} \ No newline at end of file diff --git a/experiments/think-d12-r11-run1/base_checkpoints/model_000500.pt b/experiments/think-d12-r11-run1/base_checkpoints/model_000500.pt new file mode 100644 index 0000000000000000000000000000000000000000..68a45ed0848cf9788d5d9f1e989e44c3a134972c --- /dev/null +++ b/experiments/think-d12-r11-run1/base_checkpoints/model_000500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f163e0f3998bd8d58a3484d103ce027036032d9a205b5a3ef38864f557fb1fc5 +size 792761690 diff --git a/experiments/think-d12-r11-run1/base_checkpoints/model_001000.pt b/experiments/think-d12-r11-run1/base_checkpoints/model_001000.pt new file mode 100644 index 0000000000000000000000000000000000000000..e2f872cc4369fd8999f15d7e35f4d0eac38c2a28 --- /dev/null +++ b/experiments/think-d12-r11-run1/base_checkpoints/model_001000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a962593d363a9329bffb69a38ffc18152eb473c2bf2c93257342ab5df8af24c8 +size 792761690 diff --git a/experiments/think-d12-r11-run1/base_checkpoints/model_001500.pt b/experiments/think-d12-r11-run1/base_checkpoints/model_001500.pt new file mode 100644 index 0000000000000000000000000000000000000000..d4c95927dcab99fb99fb1a191f8ff1a72794f0db --- /dev/null +++ b/experiments/think-d12-r11-run1/base_checkpoints/model_001500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:caa2774af1ed018d45043681fa8062532c12182b9e9c998b0b8db34c0dc62363 +size 792761690 diff --git a/experiments/think-d12-r11-run1/base_checkpoints/model_002000.pt b/experiments/think-d12-r11-run1/base_checkpoints/model_002000.pt new file mode 100644 index 0000000000000000000000000000000000000000..6029e2c269da256f74ca42faeedb7fe6c9e8fece --- /dev/null +++ b/experiments/think-d12-r11-run1/base_checkpoints/model_002000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bdebf84a71e02ee2a66c76258a1371c6e3cbcba232b2fd2477b4782b13e8dea4 +size 792761690 diff --git a/experiments/think-d12-r11-run1/base_checkpoints/model_002362.pt b/experiments/think-d12-r11-run1/base_checkpoints/model_002362.pt new file mode 100644 index 0000000000000000000000000000000000000000..c4d4d98f0ed88996891da9ae87b843eed02738cb --- /dev/null +++ b/experiments/think-d12-r11-run1/base_checkpoints/model_002362.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8822f407b4d43983fc9d4585d3b8ed85563940ba00b23a2490f3006d3776a282 +size 792761690 diff --git a/experiments/think-d12-r11-run1/base_checkpoints/optim_000500_rank0.pt b/experiments/think-d12-r11-run1/base_checkpoints/optim_000500_rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..0dceefc2b0cffdb5716c96b5879baaabf287dfe9 --- /dev/null +++ b/experiments/think-d12-r11-run1/base_checkpoints/optim_000500_rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cf215b8cb2fc13fe79a03eebb40e2e6f322227892b2fd6212aa9891eb40a4611 +size 1246165357 diff --git a/experiments/think-d12-r11-run1/base_checkpoints/optim_001000_rank0.pt b/experiments/think-d12-r11-run1/base_checkpoints/optim_001000_rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..48c1ec3403cbbe18d9a88cf98fbec2eb521455eb --- /dev/null +++ b/experiments/think-d12-r11-run1/base_checkpoints/optim_001000_rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4ff8f324df287c17f0ca6985b499fac453f2d28f66219755e6ab39ebd1ee5e5c +size 1246165357 diff --git a/experiments/think-d12-r11-run1/base_checkpoints/optim_001500_rank0.pt b/experiments/think-d12-r11-run1/base_checkpoints/optim_001500_rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..c6cfd7975d8a783f8730be57df056a822b4e9a37 --- /dev/null +++ b/experiments/think-d12-r11-run1/base_checkpoints/optim_001500_rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3d744e7957459bb2e1ccdd5f3fc5932a4fc62d04072d531adfd8932353e22bb7 +size 1246165357 diff --git a/experiments/think-d12-r11-run1/base_checkpoints/optim_002000_rank0.pt b/experiments/think-d12-r11-run1/base_checkpoints/optim_002000_rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..268e84fa10143716258b18fc5e6802ce74cbf758 --- /dev/null +++ b/experiments/think-d12-r11-run1/base_checkpoints/optim_002000_rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d2950ece9fc33659f8ee2a229d702ecce584fceb294345a14a0e5f26721fab1e +size 1246165357 diff --git a/experiments/think-d12-r11-run1/base_checkpoints/optim_002362_rank0.pt b/experiments/think-d12-r11-run1/base_checkpoints/optim_002362_rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..5b147787e2a81471964a57ba7166a3fdea511b16 --- /dev/null +++ b/experiments/think-d12-r11-run1/base_checkpoints/optim_002362_rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:660b1d1a545db50aa1a158248270e509c4605d9824f5b3b7332a6b09c01f8141 +size 1246165357 diff --git a/experiments/think-d12-r11-run1/config.json b/experiments/think-d12-r11-run1/config.json new file mode 100644 index 0000000000000000000000000000000000000000..52e314508da60da3652457f24c73494f07cd2602 --- /dev/null +++ b/experiments/think-d12-r11-run1/config.json @@ -0,0 +1,56 @@ +{ + "schema_version": 1, + "stage": "base", + "experiment_id": "think-d12-r11-run1", + "dataset": { + "adapter": "parquet_shards", + "repo": "jbduran/think-dataset", + "revision": "main", + "validation_shard": 472, + "num_train_shards": 24, + "download_workers": 4 + }, + "tokenizer": { + "mode": "train", + "max_chars": 2000000000, + "doc_cap": 10000, + "vocab_size": 32768 + }, + "pretokenize": { + "enabled": true, + "slack": 1.03, + "val_tokens": 20971520, + "shard_tokens": 100000000, + "tokenizer_threads": 8 + }, + "training": { + "depth": 12, + "scaling_params": 110100912, + "target_param_data_ratio": 11.25, + "window_pattern": "L", + "device_batch_size": 16, + "total_batch_size": 524288, + "seed": 42, + "save_every": 500, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "sample_every": -1 + }, + "artifacts": { + "repo": "jbduran/think.nano" + }, + "wandb": { + "entity": "jbduran-thinkingmachinesncsu", + "project": "think.nano", + "name": "think-d12-r11.25-run1", + "group": "think-d12", + "tags": [ + "think-dataset", + "d12", + "ratio11.25" + ] + }, + "config_fingerprint": "56aa9474a0ebe6a9", + "artifact_path": "experiments/think-d12-r11-run1" +} diff --git a/experiments/think-d12-r11-run1/evals/core.json b/experiments/think-d12-r11-run1/evals/core.json new file mode 100644 index 0000000000000000000000000000000000000000..639e54e5cc10296713f06cb6d2182a6f7cf2a1ad --- /dev/null +++ b/experiments/think-d12-r11-run1/evals/core.json @@ -0,0 +1,56 @@ +{ + "model": "base_model (step 2362)", + "step": 2362, + "bpb": {}, + "core_metric": 0.0791723549916879, + "core_results": { + "hellaswag_zeroshot": 0.2768372893333435, + "jeopardy": 0.0004723665479104966, + "bigbench_qa_wikidata": 0.08277151733636856, + "arc_easy": 0.31186866760253906, + "arc_challenge": 0.20392490923404694, + "copa": 0.550000011920929, + "commonsense_qa": 0.31449630856513977, + "piqa": 0.5484221577644348, + "openbook_qa": 0.26200002431869507, + "lambada_openai": 0.24956335127353668, + "hellaswag": 0.28062137961387634, + "winograd": 0.5494505763053894, + "winogrande": 0.5019731521606445, + "bigbench_dyck_languages": 0.10900000482797623, + "agi_eval_lsat_ar": 0.27391302585601807, + "bigbench_cs_algorithms": 0.4219696819782257, + "bigbench_operators": 0.06666667014360428, + "bigbench_repeat_copy_logic": 0.0, + "squad": 0.02071901597082615, + "coqa": 0.07027433067560196, + "boolq": 0.5798165202140808, + "bigbench_language_identification": 0.25209999084472656 + }, + "centered_results": { + "hellaswag_zeroshot": 0.03578305244445801, + "jeopardy": 0.0004723665479104966, + "bigbench_qa_wikidata": 0.08277151733636856, + "arc_easy": 0.08249155680338542, + "arc_challenge": -0.06143345435460409, + "copa": 0.10000002384185791, + "commonsense_qa": 0.14312038570642469, + "piqa": 0.09684431552886963, + "openbook_qa": 0.016000032424926758, + "lambada_openai": 0.24956335127353668, + "hellaswag": 0.04082850615183512, + "winograd": 0.09890115261077881, + "winogrande": 0.0039463043212890625, + "bigbench_dyck_languages": 0.10900000482797623, + "agi_eval_lsat_ar": 0.09239128232002257, + "bigbench_cs_algorithms": 0.4219696819782257, + "bigbench_operators": 0.06666667014360428, + "bigbench_repeat_copy_logic": 0.0, + "squad": 0.02071901597082615, + "coqa": 0.07027433067560196, + "boolq": -0.10574599943662943, + "bigbench_language_identification": 0.17722771270046928 + }, + "conditioned_samples": [], + "unconditioned_samples": [] +} \ No newline at end of file diff --git a/experiments/think-d12-r11-run1/evals/samples.json b/experiments/think-d12-r11-run1/evals/samples.json new file mode 100644 index 0000000000000000000000000000000000000000..ecaf279459ce24e7a254328edb339c9a5fe30d74 --- /dev/null +++ b/experiments/think-d12-r11-run1/evals/samples.json @@ -0,0 +1,48 @@ +{ + "model": "base_model (step 2362)", + "step": 2362, + "bpb": {}, + "core_metric": null, + "core_results": null, + "centered_results": null, + "conditioned_samples": [ + { + "prompt": "The capital of France is", + "text": "<|bos|>The capital of France is the capital of the United States. \n\nThe capital of the United States is the" + }, + { + "prompt": "The chemical symbol of gold is", + "text": "<|bos|>The chemical symbol of gold is the symbol of the power of the sun. The symbol of the power of the" + }, + { + "prompt": "If yesterday was Friday, then tomorrow will be", + "text": "<|bos|>If yesterday was Friday, then tomorrow will be Saturday. \n\nI am, my dear Sir, your most obedient servant,\n\nJOHN" + }, + { + "prompt": "The opposite of hot is", + "text": "<|bos|>The opposite of hot is the opposite of cold. \n\nThe opposite of cold is the opposite of heat." + }, + { + "prompt": "The planets of the solar system are:", + "text": "<|bos|>The planets of the solar system are: \n\n1. The sun, the moon, and the stars. \n\n2." + }, + { + "prompt": "My favorite color is", + "text": "<|bos|>My favorite color is the color of the sky, and the color of the earth. \n\nThe color" + }, + { + "prompt": "If 5*x + 3 = 13, then x is", + "text": "<|bos|>If 5*x + 3 = 13, then x is the same as 2, and the same as 2, and the same" + } + ], + "unconditioned_samples": [ + "<|bos|>HIERONYMUS. \n\nANDOVER-HARVARD THEOLOGICAL LIBRARY M. D. \n\n1906\n\n\u0f5d \u0f0b 70 .75 13\n\nce\n\n3\n\nDead Nature's Fire \n\nTens of coat between \u25a1 different degrees of m are; peannbs meteureux differing Raphaelite sagacity. * c missavit besides sejur de stala.\"-Mr. Herschel reported that the sun has no capability in colors, and He says: \u2191 \u221e: Single sometimes sing near together a body of the same size and appearance as the oaken staff of the sun, as if vocal in color, not", + "<|bos|>370 \n\nPaceouddy river. \n\nOne pious canoe, one-To make a dozen different animals. \n\nTwo three elephants. \n\nOne a trunk in blossom, one in spring; but two different empty eggs. 73\n\n1748] \n\nPARATIAN EPIPHANES \n\nOne large flock of dogs, one sheep, A camel in spring; one little bird, one bear. \n\nTwo jackal at three angles, one at three; one bold capmaker at one see regular exercise. \n\nTwo preserved sprigs of honey and an apple, with four dessert and melons.", + "<|bos|>Frrep rep\u00faitentot's excuse on the Si les.\"-Mure in Kerridge's Essays, Note B.-B. Blanquet. By Amyot.--A true account of the publicity, editing, and journeys of the parties \n\nWith Mayanned Notes. - FROM THE LIFE OF LEOPOLD DE GONTIER. Selected \n\n&c. &c. Printed for J. O. and R. Reston, London, 1824.\n\nF. SEIN, & CO. MCMILES, PRINTERS, NATIONAL ELECTROTYPED AND PUBLISHED. \n\n364 pp., lieving\n\nHALE COLLEGE LIBRARY \n\nDECEMBER ", + "<|bos|>HENRY MARTINEAU FOREMAN THE PERSECUTED SCHOOL \n\nMarch 23 \n\nSilk-festing the Anatomy of the Mohegans \n\nJuly 15 \n\nLetter from Mr. ALICE FROST, the Rev. Dr. SAMUEL FARNSWORTH, the Rev. Gouverneur \n\nin confidence with St. Mary Evangelist, and Dr. Seth \n\nMarch 4 Error in books, timental Library, for (Georgics, I.) 1751 page 4....... 3.......... 93 1201.)\n\nMODERN knows lies paper modesty, taking a lead to preserve us from read-\n\nooms inaccuracy in silence", + "<|bos|> HOUSE OF THE FIRST RIOTS. \n\nAgnosticism.] One who hopes to get a reputation for know- \n\n1 My Last in a Garden. I reserve my criticism of the boy in far less compatible terms than in ancient philosophy one who has hitherto trusted himself to generals. Humanitarians, by means of unphilosophical instruction, are better able to work miracles out of scepticism, scepticism, than sentimental men whose attitude towards religion could no longer be tolerated. \n\nThe lasting consequence is, that ignorance, which survives long, goes far behind the doctrine of miracle. The emotional crisis may actually be reached by absolute faith and resignation on the part", + "<|bos|>GOD AND HIS RACE. By FRANK KING NOTES FIRST NEW MEXICO. By G. H. CARY COOLIDGE \n\n(8) With an Introduction by J. F. EGAN. New York, 1900:\n\nTHE FISHERMAN. He appears in a Primer which has been compiled from \n\nProf. Hovenden's book of Professor Smith's First Principles of Geology for Research in Geology. With an Introduction by R. R. CLAY, LL.D., President of the Royal Institution. New York: \n\n1902:\n\nAN INTRODUCTION TO THE STUDY OF GEOLOGY. \n\nBy Various Authors, with a Commentary by Arthur O The Queen and her Character", + "<|bos|>Army Neglected: and Arkansas Army \n\nCoach. Vincy Vincy, Baltimore. 1.-2\n\n135 \n\nArmy\n\nMarch 1.-It was dreadfully unpopular with her people. Premising was a much dreaded term amongst them, it was asserted that she would place her confidence in little better hands than Gordon.\n\nWhat she regarded as dangerous was the thinly clad slave called \"Chasseur,\" another nominee. The trouble of conciliation was only occasionally perceptible, and, as we shall see, the Southern States refused to listen to her proposition. So that soon the situation became such that they had", + "<|bos|>the eight (1) of W\u3001 lerie w conferring constructe them-Laura de la \n\nChant, von ipol\u00f2 never' and never assures give all he has ever seen this prid\u00e8, e.'d uentechem. up half ator sau 119) 3 la raffment, non fu consolt8 f the three triumphs raf 8' i alone uaugrat student nonsense (af 9:1)!\" \u2022 cable fu \n\n12 W e barbicot, alki manton apparently improvement (nushe, waers f" + ] +} \ No newline at end of file diff --git a/experiments/think-d12-r11-run1/evals/val_bpb.json b/experiments/think-d12-r11-run1/evals/val_bpb.json new file mode 100644 index 0000000000000000000000000000000000000000..991cfe4811c5f1c9ef28f5204e86aff644f58563 --- /dev/null +++ b/experiments/think-d12-r11-run1/evals/val_bpb.json @@ -0,0 +1,12 @@ +{ + "model": "base_model (step 2362)", + "step": 2362, + "bpb": { + "val": 1.051975107450783 + }, + "core_metric": null, + "core_results": null, + "centered_results": null, + "conditioned_samples": [], + "unconditioned_samples": [] +} \ No newline at end of file diff --git a/experiments/think-d12-r11-run1/run.json b/experiments/think-d12-r11-run1/run.json new file mode 100644 index 0000000000000000000000000000000000000000..fa4db302da7e9afc548dfb559acbcddf053171ec --- /dev/null +++ b/experiments/think-d12-r11-run1/run.json @@ -0,0 +1,10 @@ +{ + "experiment_id": "think-d12-r11-run1", + "stage": "base", + "base_experiment_id": "think-d12-r11-run1", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "config_fingerprint": "56aa9474a0ebe6a9", + "wandb_run_id": "5ffd9e1f", + "created_at": 1781891761 +} diff --git a/experiments/think-d12-r11-run1/summary.json b/experiments/think-d12-r11-run1/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..f6d732a33c6672414394c897c0d5f1a0d6b717de --- /dev/null +++ b/experiments/think-d12-r11-run1/summary.json @@ -0,0 +1,93 @@ +{ + "experiment_id": "think-d12-r11-run1", + "stage": "base", + "base_experiment_id": "think-d12-r11-run1", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "dataset": "jbduran/think-dataset", + "dataset_revision": "main", + "step": 2362, + "depth": 12, + "target_param_data_ratio": 11.25, + "training_tokens": 1238368256, + "final_sampled_val_bpb": 1.1026519310163985, + "minimum_sampled_val_bpb": 1.1026519310163985, + "full_val_bpb": 1.051975107450783, + "core_metric": 0.0791723549916879, + "centered_results": { + "hellaswag_zeroshot": 0.03578305244445801, + "jeopardy": 0.0004723665479104966, + "bigbench_qa_wikidata": 0.08277151733636856, + "arc_easy": 0.08249155680338542, + "arc_challenge": -0.06143345435460409, + "copa": 0.10000002384185791, + "commonsense_qa": 0.14312038570642469, + "piqa": 0.09684431552886963, + "openbook_qa": 0.016000032424926758, + "lambada_openai": 0.24956335127353668, + "hellaswag": 0.04082850615183512, + "winograd": 0.09890115261077881, + "winogrande": 0.0039463043212890625, + "bigbench_dyck_languages": 0.10900000482797623, + "agi_eval_lsat_ar": 0.09239128232002257, + "bigbench_cs_algorithms": 0.4219696819782257, + "bigbench_operators": 0.06666667014360428, + "bigbench_repeat_copy_logic": 0.0, + "squad": 0.02071901597082615, + "coqa": 0.07027433067560196, + "boolq": -0.10574599943662943, + "bigbench_language_identification": 0.17722771270046928 + }, + "conditioned_samples": [ + { + "prompt": "The capital of France is", + "text": "<|bos|>The capital of France is the capital of the United States. \n\nThe capital of the United States is the" + }, + { + "prompt": "The chemical symbol of gold is", + "text": "<|bos|>The chemical symbol of gold is the symbol of the power of the sun. The symbol of the power of the" + }, + { + "prompt": "If yesterday was Friday, then tomorrow will be", + "text": "<|bos|>If yesterday was Friday, then tomorrow will be Saturday. \n\nI am, my dear Sir, your most obedient servant,\n\nJOHN" + }, + { + "prompt": "The opposite of hot is", + "text": "<|bos|>The opposite of hot is the opposite of cold. \n\nThe opposite of cold is the opposite of heat." + }, + { + "prompt": "The planets of the solar system are:", + "text": "<|bos|>The planets of the solar system are: \n\n1. The sun, the moon, and the stars. \n\n2." + }, + { + "prompt": "My favorite color is", + "text": "<|bos|>My favorite color is the color of the sky, and the color of the earth. \n\nThe color" + }, + { + "prompt": "If 5*x + 3 = 13, then x is", + "text": "<|bos|>If 5*x + 3 = 13, then x is the same as 2, and the same as 2, and the same" + } + ], + "unconditioned_samples": [ + "<|bos|>HIERONYMUS. \n\nANDOVER-HARVARD THEOLOGICAL LIBRARY M. D. \n\n1906\n\n\u0f5d \u0f0b 70 .75 13\n\nce\n\n3\n\nDead Nature's Fire \n\nTens of coat between \u25a1 different degrees of m are; peannbs meteureux differing Raphaelite sagacity. * c missavit besides sejur de stala.\"-Mr. Herschel reported that the sun has no capability in colors, and He says: \u2191 \u221e: Single sometimes sing near together a body of the same size and appearance as the oaken staff of the sun, as if vocal in color, not", + "<|bos|>370 \n\nPaceouddy river. \n\nOne pious canoe, one-To make a dozen different animals. \n\nTwo three elephants. \n\nOne a trunk in blossom, one in spring; but two different empty eggs. 73\n\n1748] \n\nPARATIAN EPIPHANES \n\nOne large flock of dogs, one sheep, A camel in spring; one little bird, one bear. \n\nTwo jackal at three angles, one at three; one bold capmaker at one see regular exercise. \n\nTwo preserved sprigs of honey and an apple, with four dessert and melons.", + "<|bos|>Frrep rep\u00faitentot's excuse on the Si les.\"-Mure in Kerridge's Essays, Note B.-B. Blanquet. By Amyot.--A true account of the publicity, editing, and journeys of the parties \n\nWith Mayanned Notes. - FROM THE LIFE OF LEOPOLD DE GONTIER. Selected \n\n&c. &c. Printed for J. O. and R. Reston, London, 1824.\n\nF. SEIN, & CO. MCMILES, PRINTERS, NATIONAL ELECTROTYPED AND PUBLISHED. \n\n364 pp., lieving\n\nHALE COLLEGE LIBRARY \n\nDECEMBER ", + "<|bos|>HENRY MARTINEAU FOREMAN THE PERSECUTED SCHOOL \n\nMarch 23 \n\nSilk-festing the Anatomy of the Mohegans \n\nJuly 15 \n\nLetter from Mr. ALICE FROST, the Rev. Dr. SAMUEL FARNSWORTH, the Rev. Gouverneur \n\nin confidence with St. Mary Evangelist, and Dr. Seth \n\nMarch 4 Error in books, timental Library, for (Georgics, I.) 1751 page 4....... 3.......... 93 1201.)\n\nMODERN knows lies paper modesty, taking a lead to preserve us from read-\n\nooms inaccuracy in silence", + "<|bos|> HOUSE OF THE FIRST RIOTS. \n\nAgnosticism.] One who hopes to get a reputation for know- \n\n1 My Last in a Garden. I reserve my criticism of the boy in far less compatible terms than in ancient philosophy one who has hitherto trusted himself to generals. Humanitarians, by means of unphilosophical instruction, are better able to work miracles out of scepticism, scepticism, than sentimental men whose attitude towards religion could no longer be tolerated. \n\nThe lasting consequence is, that ignorance, which survives long, goes far behind the doctrine of miracle. The emotional crisis may actually be reached by absolute faith and resignation on the part", + "<|bos|>GOD AND HIS RACE. By FRANK KING NOTES FIRST NEW MEXICO. By G. H. CARY COOLIDGE \n\n(8) With an Introduction by J. F. EGAN. New York, 1900:\n\nTHE FISHERMAN. He appears in a Primer which has been compiled from \n\nProf. Hovenden's book of Professor Smith's First Principles of Geology for Research in Geology. With an Introduction by R. R. CLAY, LL.D., President of the Royal Institution. New York: \n\n1902:\n\nAN INTRODUCTION TO THE STUDY OF GEOLOGY. \n\nBy Various Authors, with a Commentary by Arthur O The Queen and her Character", + "<|bos|>Army Neglected: and Arkansas Army \n\nCoach. Vincy Vincy, Baltimore. 1.-2\n\n135 \n\nArmy\n\nMarch 1.-It was dreadfully unpopular with her people. Premising was a much dreaded term amongst them, it was asserted that she would place her confidence in little better hands than Gordon.\n\nWhat she regarded as dangerous was the thinly clad slave called \"Chasseur,\" another nominee. The trouble of conciliation was only occasionally perceptible, and, as we shall see, the Southern States refused to listen to her proposition. So that soon the situation became such that they had", + "<|bos|>the eight (1) of W\u3001 lerie w conferring constructe them-Laura de la \n\nChant, von ipol\u00f2 never' and never assures give all he has ever seen this prid\u00e8, e.'d uentechem. up half ator sau 119) 3 la raffment, non fu consolt8 f the three triumphs raf 8' i alone uaugrat student nonsense (af 9:1)!\" \u2022 cable fu \n\n12 W e barbicot, alki manton apparently improvement (nushe, waers f" + ], + "training_time_seconds": 6249.3527302742, + "stage_training_flops": 1.0985538644638433e+18, + "inherited_parent_flops": 0.0, + "cumulative_pipeline_training_flops": 1.0985538644638433e+18, + "config_fingerprint": "56aa9474a0ebe6a9", + "git_commit_sha": "4d4b7c5e2a8f77393c2c541587f4cd311b7122e0", + "wandb_url": "https://wandb.ai/jbduran-thinkingmachinesncsu/think.nano/runs/5ffd9e1f", + "huggingface_url": "https://huggingface.co/jbduran/think.nano/tree/main/experiments/think-d12-r11-run1", + "dataset_fingerprint": "63a5e6be81591d82", + "tokenizer_fingerprint": "5a7fc542f0b39fb9", + "unique_train_tokens": 1275519304, + "effective_epochs": 0.970873786164196 +} diff --git a/experiments/think-d12-r11-run1/tokenizer/experiment_tokenizer.json b/experiments/think-d12-r11-run1/tokenizer/experiment_tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..d2ae30a5932f947ce26b87eca8fc3bbb83ddf0cf --- /dev/null +++ b/experiments/think-d12-r11-run1/tokenizer/experiment_tokenizer.json @@ -0,0 +1,18 @@ +{ + "experiment_id": "think-d12-r11-run1", + "dataset": { + "adapter": "parquet_shards", + "repo": "jbduran/think-dataset", + "revision": "main", + "validation_shard": 472, + "num_train_shards": 24, + "download_workers": 4 + }, + "tokenizer": { + "mode": "train", + "max_chars": 2000000000, + "doc_cap": 10000, + "vocab_size": 32768 + }, + "created_at": 1781890485 +} diff --git a/experiments/think-d12-r11-run1/tokenizer/token_bytes.pt b/experiments/think-d12-r11-run1/tokenizer/token_bytes.pt new file mode 100644 index 0000000000000000000000000000000000000000..01d1ec4aab9e8a7d205c3b3ffbeb8da0e9a62db1 --- /dev/null +++ b/experiments/think-d12-r11-run1/tokenizer/token_bytes.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:53db25a1f6969198e67c303e479e97fb49859c715061baec525471666a2f47e1 +size 132649 diff --git a/experiments/think-d12-r11-run1/tokenizer/tokenizer.pkl b/experiments/think-d12-r11-run1/tokenizer/tokenizer.pkl new file mode 100644 index 0000000000000000000000000000000000000000..a17bd392980021628053b95d6425fc556aad527a --- /dev/null +++ b/experiments/think-d12-r11-run1/tokenizer/tokenizer.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:42ea52179dcefb47e4a4bbca7c37f67c3373840a3681587b1c8fe188f00d3bf1 +size 404071 diff --git a/experiments/think-d12-r11-run2/base_checkpoints/meta_000500.json b/experiments/think-d12-r11-run2/base_checkpoints/meta_000500.json new file mode 100644 index 0000000000000000000000000000000000000000..123fa7415b3bf8a9adc7826f66f4e1747d962624 --- /dev/null +++ b/experiments/think-d12-r11-run2/base_checkpoints/meta_000500.json @@ -0,0 +1,138 @@ +{ + "step": 500, + "experiment_id": "think-d12-r11-run2", + "val_bpb": 1.3256252804681699, + "model_config": { + "sequence_len": 2048, + "vocab_size": 32768, + "n_layer": 12, + "n_head": 6, + "n_kv_head": 6, + "n_embd": 768, + "window_pattern": "L" + }, + "user_config": { + "run": "think-d12-r11.25-run2", + "wandb_run_id": "84e31c8e", + "wandb_group": "think-d12", + "wandb_tags": "think-dataset,d12,ratio11.25", + "device_type": "", + "fp8": false, + "fp8_recipe": "tensorwise", + "depth": 12, + "aspect_ratio": 64, + "head_dim": 128, + "max_seq_len": 2048, + "window_pattern": "L", + "num_iterations": -1, + "target_flops": -1.0, + "target_param_data_ratio": 11.25, + "device_batch_size": 16, + "total_batch_size": 524288, + "embedding_lr": 0.3, + "unembedding_lr": 0.008, + "weight_decay": 0.28, + "matrix_lr": 0.02, + "scalar_lr": 0.5, + "warmup_steps": 40, + "warmdown_ratio": 0.65, + "final_lr_frac": 0.05, + "resume_from_step": -1, + "pretokenized": true, + "data_dir": "/content/nanochat_cache/experiments/think-d12-r11-run2/data", + "tokenizer_dir": "/content/nanochat_cache/experiments/think-d12-r11-run2/tokenizer", + "pretokenized_dir": "/content/nanochat_cache/experiments/think-d12-r11-run2/pretok", + "checkpoint_dir": "/content/nanochat_cache/experiments/think-d12-r11-run2/base_checkpoints", + "experiment_id": "think-d12-r11-run2", + "experiment_config": "/content/nanochat_cache/experiments/think-d12-r11-run2/config.json", + "tokenizer_fingerprint": "96dd8e502a849d59", + "git_commit_sha": "48cd5f516123fcf119c7a43a32dd1dfd5c7b8eaa", + "seed": 43, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "core_metric_max_per_task": 500, + "sample_every": -1, + "save_every": 500, + "model_tag": "think-d12-r11-run2", + "experiment": { + "schema_version": 1, + "stage": "base", + "experiment_id": "think-d12-r11-run2", + "dataset": { + "adapter": "parquet_shards", + "repo": "jbduran/think-dataset", + "revision": "main", + "validation_shard": 472, + "num_train_shards": 24, + "download_workers": 4 + }, + "tokenizer": { + "mode": "train", + "max_chars": 2000000000, + "doc_cap": 10000, + "vocab_size": 32768 + }, + "pretokenize": { + "enabled": true, + "slack": 1.03, + "val_tokens": 20971520, + "shard_tokens": 100000000, + "tokenizer_threads": 8 + }, + "training": { + "depth": 12, + "scaling_params": 110100912, + "target_param_data_ratio": 11.25, + "window_pattern": "L", + "device_batch_size": 16, + "total_batch_size": 524288, + "seed": 43, + "save_every": 500, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "sample_every": -1 + }, + "artifacts": { + "repo": "jbduran/think.nano" + }, + "wandb": { + "entity": "jbduran-thinkingmachinesncsu", + "project": "think.nano", + "name": "think-d12-r11.25-run2", + "group": "think-d12", + "tags": [ + "think-dataset", + "d12", + "ratio11.25" + ] + }, + "config_fingerprint": "7f54bf07c5352d97", + "artifact_path": "experiments/think-d12-r11-run2" + }, + "stage": "base", + "base_experiment_id": "think-d12-r11-run2", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "config_fingerprint": "7f54bf07c5352d97" + }, + "device_batch_size": 16, + "max_seq_len": 2048, + "total_batch_size": 524288, + "dataloader_state_dict": { + "file_idx": 2, + "pos": 62184769, + "epoch": 1, + "pq_idx": 2, + "rg_idx": 62184769 + }, + "loop_state": { + "min_val_bpb": 1.3256252804681699, + "smooth_train_loss": 3.588497145795441, + "total_training_time": 1296.8187935352325, + "stage_training_flops": 232547388751872000, + "inherited_parent_flops": 0.0, + "cumulative_pipeline_training_flops": 232547388751872000 + } +} \ No newline at end of file diff --git a/experiments/think-d12-r11-run2/base_checkpoints/meta_001000.json b/experiments/think-d12-r11-run2/base_checkpoints/meta_001000.json new file mode 100644 index 0000000000000000000000000000000000000000..6c17fb2048fec39c11242179c5324a14ad293552 --- /dev/null +++ b/experiments/think-d12-r11-run2/base_checkpoints/meta_001000.json @@ -0,0 +1,138 @@ +{ + "step": 1000, + "experiment_id": "think-d12-r11-run2", + "val_bpb": 1.2351307837396184, + "model_config": { + "sequence_len": 2048, + "vocab_size": 32768, + "n_layer": 12, + "n_head": 6, + "n_kv_head": 6, + "n_embd": 768, + "window_pattern": "L" + }, + "user_config": { + "run": "think-d12-r11.25-run2", + "wandb_run_id": "84e31c8e", + "wandb_group": "think-d12", + "wandb_tags": "think-dataset,d12,ratio11.25", + "device_type": "", + "fp8": false, + "fp8_recipe": "tensorwise", + "depth": 12, + "aspect_ratio": 64, + "head_dim": 128, + "max_seq_len": 2048, + "window_pattern": "L", + "num_iterations": -1, + "target_flops": -1.0, + "target_param_data_ratio": 11.25, + "device_batch_size": 16, + "total_batch_size": 524288, + "embedding_lr": 0.3, + "unembedding_lr": 0.008, + "weight_decay": 0.28, + "matrix_lr": 0.02, + "scalar_lr": 0.5, + "warmup_steps": 40, + "warmdown_ratio": 0.65, + "final_lr_frac": 0.05, + "resume_from_step": -1, + "pretokenized": true, + "data_dir": "/content/nanochat_cache/experiments/think-d12-r11-run2/data", + "tokenizer_dir": "/content/nanochat_cache/experiments/think-d12-r11-run2/tokenizer", + "pretokenized_dir": "/content/nanochat_cache/experiments/think-d12-r11-run2/pretok", + "checkpoint_dir": "/content/nanochat_cache/experiments/think-d12-r11-run2/base_checkpoints", + "experiment_id": "think-d12-r11-run2", + "experiment_config": "/content/nanochat_cache/experiments/think-d12-r11-run2/config.json", + "tokenizer_fingerprint": "96dd8e502a849d59", + "git_commit_sha": "48cd5f516123fcf119c7a43a32dd1dfd5c7b8eaa", + "seed": 43, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "core_metric_max_per_task": 500, + "sample_every": -1, + "save_every": 500, + "model_tag": "think-d12-r11-run2", + "experiment": { + "schema_version": 1, + "stage": "base", + "experiment_id": "think-d12-r11-run2", + "dataset": { + "adapter": "parquet_shards", + "repo": "jbduran/think-dataset", + "revision": "main", + "validation_shard": 472, + "num_train_shards": 24, + "download_workers": 4 + }, + "tokenizer": { + "mode": "train", + "max_chars": 2000000000, + "doc_cap": 10000, + "vocab_size": 32768 + }, + "pretokenize": { + "enabled": true, + "slack": 1.03, + "val_tokens": 20971520, + "shard_tokens": 100000000, + "tokenizer_threads": 8 + }, + "training": { + "depth": 12, + "scaling_params": 110100912, + "target_param_data_ratio": 11.25, + "window_pattern": "L", + "device_batch_size": 16, + "total_batch_size": 524288, + "seed": 43, + "save_every": 500, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "sample_every": -1 + }, + "artifacts": { + "repo": "jbduran/think.nano" + }, + "wandb": { + "entity": "jbduran-thinkingmachinesncsu", + "project": "think.nano", + "name": "think-d12-r11.25-run2", + "group": "think-d12", + "tags": [ + "think-dataset", + "d12", + "ratio11.25" + ] + }, + "config_fingerprint": "7f54bf07c5352d97", + "artifact_path": "experiments/think-d12-r11-run2" + }, + "stage": "base", + "base_experiment_id": "think-d12-r11-run2", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "config_fingerprint": "7f54bf07c5352d97" + }, + "device_batch_size": 16, + "max_seq_len": 2048, + "total_batch_size": 524288, + "dataloader_state_dict": { + "file_idx": 5, + "pos": 24336769, + "epoch": 1, + "pq_idx": 5, + "rg_idx": 24336769 + }, + "loop_state": { + "min_val_bpb": 1.2351307837396184, + "smooth_train_loss": 3.4494649735291096, + "total_training_time": 2619.1334071159363, + "stage_training_flops": 465094777503744000, + "inherited_parent_flops": 0.0, + "cumulative_pipeline_training_flops": 465094777503744000 + } +} \ No newline at end of file diff --git a/experiments/think-d12-r11-run2/base_checkpoints/meta_001500.json b/experiments/think-d12-r11-run2/base_checkpoints/meta_001500.json new file mode 100644 index 0000000000000000000000000000000000000000..d3d846e146e47cb84191d7938ffef78dfd8e2a87 --- /dev/null +++ b/experiments/think-d12-r11-run2/base_checkpoints/meta_001500.json @@ -0,0 +1,138 @@ +{ + "step": 1500, + "experiment_id": "think-d12-r11-run2", + "val_bpb": 1.1760284474435965, + "model_config": { + "sequence_len": 2048, + "vocab_size": 32768, + "n_layer": 12, + "n_head": 6, + "n_kv_head": 6, + "n_embd": 768, + "window_pattern": "L" + }, + "user_config": { + "run": "think-d12-r11.25-run2", + "wandb_run_id": "84e31c8e", + "wandb_group": "think-d12", + "wandb_tags": "think-dataset,d12,ratio11.25", + "device_type": "", + "fp8": false, + "fp8_recipe": "tensorwise", + "depth": 12, + "aspect_ratio": 64, + "head_dim": 128, + "max_seq_len": 2048, + "window_pattern": "L", + "num_iterations": -1, + "target_flops": -1.0, + "target_param_data_ratio": 11.25, + "device_batch_size": 16, + "total_batch_size": 524288, + "embedding_lr": 0.3, + "unembedding_lr": 0.008, + "weight_decay": 0.28, + "matrix_lr": 0.02, + "scalar_lr": 0.5, + "warmup_steps": 40, + "warmdown_ratio": 0.65, + "final_lr_frac": 0.05, + "resume_from_step": -1, + "pretokenized": true, + "data_dir": "/content/nanochat_cache/experiments/think-d12-r11-run2/data", + "tokenizer_dir": "/content/nanochat_cache/experiments/think-d12-r11-run2/tokenizer", + "pretokenized_dir": "/content/nanochat_cache/experiments/think-d12-r11-run2/pretok", + "checkpoint_dir": "/content/nanochat_cache/experiments/think-d12-r11-run2/base_checkpoints", + "experiment_id": "think-d12-r11-run2", + "experiment_config": "/content/nanochat_cache/experiments/think-d12-r11-run2/config.json", + "tokenizer_fingerprint": "96dd8e502a849d59", + "git_commit_sha": "48cd5f516123fcf119c7a43a32dd1dfd5c7b8eaa", + "seed": 43, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "core_metric_max_per_task": 500, + "sample_every": -1, + "save_every": 500, + "model_tag": "think-d12-r11-run2", + "experiment": { + "schema_version": 1, + "stage": "base", + "experiment_id": "think-d12-r11-run2", + "dataset": { + "adapter": "parquet_shards", + "repo": "jbduran/think-dataset", + "revision": "main", + "validation_shard": 472, + "num_train_shards": 24, + "download_workers": 4 + }, + "tokenizer": { + "mode": "train", + "max_chars": 2000000000, + "doc_cap": 10000, + "vocab_size": 32768 + }, + "pretokenize": { + "enabled": true, + "slack": 1.03, + "val_tokens": 20971520, + "shard_tokens": 100000000, + "tokenizer_threads": 8 + }, + "training": { + "depth": 12, + "scaling_params": 110100912, + "target_param_data_ratio": 11.25, + "window_pattern": "L", + "device_batch_size": 16, + "total_batch_size": 524288, + "seed": 43, + "save_every": 500, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "sample_every": -1 + }, + "artifacts": { + "repo": "jbduran/think.nano" + }, + "wandb": { + "entity": "jbduran-thinkingmachinesncsu", + "project": "think.nano", + "name": "think-d12-r11.25-run2", + "group": "think-d12", + "tags": [ + "think-dataset", + "d12", + "ratio11.25" + ] + }, + "config_fingerprint": "7f54bf07c5352d97", + "artifact_path": "experiments/think-d12-r11-run2" + }, + "stage": "base", + "base_experiment_id": "think-d12-r11-run2", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "config_fingerprint": "7f54bf07c5352d97" + }, + "device_batch_size": 16, + "max_seq_len": 2048, + "total_batch_size": 524288, + "dataloader_state_dict": { + "file_idx": 7, + "pos": 86488769, + "epoch": 1, + "pq_idx": 7, + "rg_idx": 86488769 + }, + "loop_state": { + "min_val_bpb": 1.1760284474435965, + "smooth_train_loss": 3.247025462772539, + "total_training_time": 3941.2222929000854, + "stage_training_flops": 697642166255616000, + "inherited_parent_flops": 0.0, + "cumulative_pipeline_training_flops": 697642166255616000 + } +} \ No newline at end of file diff --git a/experiments/think-d12-r11-run2/base_checkpoints/meta_002000.json b/experiments/think-d12-r11-run2/base_checkpoints/meta_002000.json new file mode 100644 index 0000000000000000000000000000000000000000..d160329610ec65c61ca10114fa1e213f9086213f --- /dev/null +++ b/experiments/think-d12-r11-run2/base_checkpoints/meta_002000.json @@ -0,0 +1,138 @@ +{ + "step": 2000, + "experiment_id": "think-d12-r11-run2", + "val_bpb": 1.1227598491207411, + "model_config": { + "sequence_len": 2048, + "vocab_size": 32768, + "n_layer": 12, + "n_head": 6, + "n_kv_head": 6, + "n_embd": 768, + "window_pattern": "L" + }, + "user_config": { + "run": "think-d12-r11.25-run2", + "wandb_run_id": "84e31c8e", + "wandb_group": "think-d12", + "wandb_tags": "think-dataset,d12,ratio11.25", + "device_type": "", + "fp8": false, + "fp8_recipe": "tensorwise", + "depth": 12, + "aspect_ratio": 64, + "head_dim": 128, + "max_seq_len": 2048, + "window_pattern": "L", + "num_iterations": -1, + "target_flops": -1.0, + "target_param_data_ratio": 11.25, + "device_batch_size": 16, + "total_batch_size": 524288, + "embedding_lr": 0.3, + "unembedding_lr": 0.008, + "weight_decay": 0.28, + "matrix_lr": 0.02, + "scalar_lr": 0.5, + "warmup_steps": 40, + "warmdown_ratio": 0.65, + "final_lr_frac": 0.05, + "resume_from_step": -1, + "pretokenized": true, + "data_dir": "/content/nanochat_cache/experiments/think-d12-r11-run2/data", + "tokenizer_dir": "/content/nanochat_cache/experiments/think-d12-r11-run2/tokenizer", + "pretokenized_dir": "/content/nanochat_cache/experiments/think-d12-r11-run2/pretok", + "checkpoint_dir": "/content/nanochat_cache/experiments/think-d12-r11-run2/base_checkpoints", + "experiment_id": "think-d12-r11-run2", + "experiment_config": "/content/nanochat_cache/experiments/think-d12-r11-run2/config.json", + "tokenizer_fingerprint": "96dd8e502a849d59", + "git_commit_sha": "48cd5f516123fcf119c7a43a32dd1dfd5c7b8eaa", + "seed": 43, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "core_metric_max_per_task": 500, + "sample_every": -1, + "save_every": 500, + "model_tag": "think-d12-r11-run2", + "experiment": { + "schema_version": 1, + "stage": "base", + "experiment_id": "think-d12-r11-run2", + "dataset": { + "adapter": "parquet_shards", + "repo": "jbduran/think-dataset", + "revision": "main", + "validation_shard": 472, + "num_train_shards": 24, + "download_workers": 4 + }, + "tokenizer": { + "mode": "train", + "max_chars": 2000000000, + "doc_cap": 10000, + "vocab_size": 32768 + }, + "pretokenize": { + "enabled": true, + "slack": 1.03, + "val_tokens": 20971520, + "shard_tokens": 100000000, + "tokenizer_threads": 8 + }, + "training": { + "depth": 12, + "scaling_params": 110100912, + "target_param_data_ratio": 11.25, + "window_pattern": "L", + "device_batch_size": 16, + "total_batch_size": 524288, + "seed": 43, + "save_every": 500, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "sample_every": -1 + }, + "artifacts": { + "repo": "jbduran/think.nano" + }, + "wandb": { + "entity": "jbduran-thinkingmachinesncsu", + "project": "think.nano", + "name": "think-d12-r11.25-run2", + "group": "think-d12", + "tags": [ + "think-dataset", + "d12", + "ratio11.25" + ] + }, + "config_fingerprint": "7f54bf07c5352d97", + "artifact_path": "experiments/think-d12-r11-run2" + }, + "stage": "base", + "base_experiment_id": "think-d12-r11-run2", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "config_fingerprint": "7f54bf07c5352d97" + }, + "device_batch_size": 16, + "max_seq_len": 2048, + "total_batch_size": 524288, + "dataloader_state_dict": { + "file_idx": 10, + "pos": 48640769, + "epoch": 1, + "pq_idx": 10, + "rg_idx": 48640769 + }, + "loop_state": { + "min_val_bpb": 1.1227598491207411, + "smooth_train_loss": 3.2449127117831824, + "total_training_time": 5264.500639915466, + "stage_training_flops": 930189555007488000, + "inherited_parent_flops": 0.0, + "cumulative_pipeline_training_flops": 930189555007488000 + } +} \ No newline at end of file diff --git a/experiments/think-d12-r11-run2/base_checkpoints/meta_002362.json b/experiments/think-d12-r11-run2/base_checkpoints/meta_002362.json new file mode 100644 index 0000000000000000000000000000000000000000..c5f7d41e8667fdf1a46a2bd5c43ff0f38cba7601 --- /dev/null +++ b/experiments/think-d12-r11-run2/base_checkpoints/meta_002362.json @@ -0,0 +1,138 @@ +{ + "step": 2362, + "experiment_id": "think-d12-r11-run2", + "val_bpb": 1.101818437651454, + "model_config": { + "sequence_len": 2048, + "vocab_size": 32768, + "n_layer": 12, + "n_head": 6, + "n_kv_head": 6, + "n_embd": 768, + "window_pattern": "L" + }, + "user_config": { + "run": "think-d12-r11.25-run2", + "wandb_run_id": "84e31c8e", + "wandb_group": "think-d12", + "wandb_tags": "think-dataset,d12,ratio11.25", + "device_type": "", + "fp8": false, + "fp8_recipe": "tensorwise", + "depth": 12, + "aspect_ratio": 64, + "head_dim": 128, + "max_seq_len": 2048, + "window_pattern": "L", + "num_iterations": -1, + "target_flops": -1.0, + "target_param_data_ratio": 11.25, + "device_batch_size": 16, + "total_batch_size": 524288, + "embedding_lr": 0.3, + "unembedding_lr": 0.008, + "weight_decay": 0.28, + "matrix_lr": 0.02, + "scalar_lr": 0.5, + "warmup_steps": 40, + "warmdown_ratio": 0.65, + "final_lr_frac": 0.05, + "resume_from_step": -1, + "pretokenized": true, + "data_dir": "/content/nanochat_cache/experiments/think-d12-r11-run2/data", + "tokenizer_dir": "/content/nanochat_cache/experiments/think-d12-r11-run2/tokenizer", + "pretokenized_dir": "/content/nanochat_cache/experiments/think-d12-r11-run2/pretok", + "checkpoint_dir": "/content/nanochat_cache/experiments/think-d12-r11-run2/base_checkpoints", + "experiment_id": "think-d12-r11-run2", + "experiment_config": "/content/nanochat_cache/experiments/think-d12-r11-run2/config.json", + "tokenizer_fingerprint": "96dd8e502a849d59", + "git_commit_sha": "48cd5f516123fcf119c7a43a32dd1dfd5c7b8eaa", + "seed": 43, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "core_metric_max_per_task": 500, + "sample_every": -1, + "save_every": 500, + "model_tag": "think-d12-r11-run2", + "experiment": { + "schema_version": 1, + "stage": "base", + "experiment_id": "think-d12-r11-run2", + "dataset": { + "adapter": "parquet_shards", + "repo": "jbduran/think-dataset", + "revision": "main", + "validation_shard": 472, + "num_train_shards": 24, + "download_workers": 4 + }, + "tokenizer": { + "mode": "train", + "max_chars": 2000000000, + "doc_cap": 10000, + "vocab_size": 32768 + }, + "pretokenize": { + "enabled": true, + "slack": 1.03, + "val_tokens": 20971520, + "shard_tokens": 100000000, + "tokenizer_threads": 8 + }, + "training": { + "depth": 12, + "scaling_params": 110100912, + "target_param_data_ratio": 11.25, + "window_pattern": "L", + "device_batch_size": 16, + "total_batch_size": 524288, + "seed": 43, + "save_every": 500, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "sample_every": -1 + }, + "artifacts": { + "repo": "jbduran/think.nano" + }, + "wandb": { + "entity": "jbduran-thinkingmachinesncsu", + "project": "think.nano", + "name": "think-d12-r11.25-run2", + "group": "think-d12", + "tags": [ + "think-dataset", + "d12", + "ratio11.25" + ] + }, + "config_fingerprint": "7f54bf07c5352d97", + "artifact_path": "experiments/think-d12-r11-run2" + }, + "stage": "base", + "base_experiment_id": "think-d12-r11-run2", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "config_fingerprint": "7f54bf07c5352d97" + }, + "device_batch_size": 16, + "max_seq_len": 2048, + "total_batch_size": 524288, + "dataloader_state_dict": { + "file_idx": 12, + "pos": 38438817, + "epoch": 1, + "pq_idx": 12, + "rg_idx": 38438817 + }, + "loop_state": { + "min_val_bpb": 1.101818437651454, + "smooth_train_loss": 3.0713499916611764, + "total_training_time": 6222.97850894928, + "stage_training_flops": 1098553864463843328, + "inherited_parent_flops": 0.0, + "cumulative_pipeline_training_flops": 1098553864463843328 + } +} \ No newline at end of file diff --git a/experiments/think-d12-r11-run2/base_checkpoints/model_000500.pt b/experiments/think-d12-r11-run2/base_checkpoints/model_000500.pt new file mode 100644 index 0000000000000000000000000000000000000000..75fc52d3ea97dd5967ef987cc73fc50b736926b4 --- /dev/null +++ b/experiments/think-d12-r11-run2/base_checkpoints/model_000500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6aa80a8d2fbc587741198baa1c23797e9a03abf47e82daa8402749cbba01ed03 +size 792761690 diff --git a/experiments/think-d12-r11-run2/base_checkpoints/model_001000.pt b/experiments/think-d12-r11-run2/base_checkpoints/model_001000.pt new file mode 100644 index 0000000000000000000000000000000000000000..0a304b79f0a6c2d2dd7919b1cf729d48ceb2f121 --- /dev/null +++ b/experiments/think-d12-r11-run2/base_checkpoints/model_001000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4068e780a404f26a487f94bb746300d3640c41ce752520be014c3518ec09e8d1 +size 792761690 diff --git a/experiments/think-d12-r11-run2/base_checkpoints/model_001500.pt b/experiments/think-d12-r11-run2/base_checkpoints/model_001500.pt new file mode 100644 index 0000000000000000000000000000000000000000..2b7d99a758bacccd3d57e3dd4bbfeae90d43da86 --- /dev/null +++ b/experiments/think-d12-r11-run2/base_checkpoints/model_001500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:afbbdc53ee7105f7e13c54cb309e88b6f1ba8a07722ba671a4e28dc0052c6f92 +size 792761690 diff --git a/experiments/think-d12-r11-run2/base_checkpoints/model_002000.pt b/experiments/think-d12-r11-run2/base_checkpoints/model_002000.pt new file mode 100644 index 0000000000000000000000000000000000000000..1080733e27e63e648d3edf94c313321989b5a699 --- /dev/null +++ b/experiments/think-d12-r11-run2/base_checkpoints/model_002000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ffc92e32294ed34f72e7bfb6fc4968eaec6df567a60723d8ad4cccf4b7228d4b +size 792761690 diff --git a/experiments/think-d12-r11-run2/base_checkpoints/model_002362.pt b/experiments/think-d12-r11-run2/base_checkpoints/model_002362.pt new file mode 100644 index 0000000000000000000000000000000000000000..13a26e86b3c1f375010c13664f2d6798a61004a2 --- /dev/null +++ b/experiments/think-d12-r11-run2/base_checkpoints/model_002362.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:48333070fea388243792d978e6a4727bc984eb0115dd735a9de5893f8aaa8e6f +size 792761690 diff --git a/experiments/think-d12-r11-run2/base_checkpoints/optim_000500_rank0.pt b/experiments/think-d12-r11-run2/base_checkpoints/optim_000500_rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..b82bd99cea63ac980474477f59f4dedb44466802 --- /dev/null +++ b/experiments/think-d12-r11-run2/base_checkpoints/optim_000500_rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:56797f52a92fbb0a9945f2fa451bdf957f6118037452b23f009acce5067c80c4 +size 1246165357 diff --git a/experiments/think-d12-r11-run2/base_checkpoints/optim_001000_rank0.pt b/experiments/think-d12-r11-run2/base_checkpoints/optim_001000_rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..3e0b9b1b09dc953fe8b21bc723cf0907440a1045 --- /dev/null +++ b/experiments/think-d12-r11-run2/base_checkpoints/optim_001000_rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6601227b18f462e666bfd581050d81675e9df2d1cb0d2a0c8342c788ae16fced +size 1246165357 diff --git a/experiments/think-d12-r11-run2/base_checkpoints/optim_001500_rank0.pt b/experiments/think-d12-r11-run2/base_checkpoints/optim_001500_rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..210fdfcaa0869930eeaf6068136348cfeebafe09 --- /dev/null +++ b/experiments/think-d12-r11-run2/base_checkpoints/optim_001500_rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b5e3172a659b7e58ffd0be60d94c7f6ea2ca39924158b4fddfc0389f7a39ef5e +size 1246165357 diff --git a/experiments/think-d12-r11-run2/base_checkpoints/optim_002000_rank0.pt b/experiments/think-d12-r11-run2/base_checkpoints/optim_002000_rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..4bc87ad14282e99f19824cfd23a102392d962b51 --- /dev/null +++ b/experiments/think-d12-r11-run2/base_checkpoints/optim_002000_rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b419389821092e526cb0a61153dcab4ebf42277a9d36a9406c7606bb790eb619 +size 1246165357 diff --git a/experiments/think-d12-r11-run2/base_checkpoints/optim_002362_rank0.pt b/experiments/think-d12-r11-run2/base_checkpoints/optim_002362_rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..6db65fbe3e595029fbe27a4a39f765184c37b30e --- /dev/null +++ b/experiments/think-d12-r11-run2/base_checkpoints/optim_002362_rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c08b338b4adcd6f31408e290e73eb41e37994c7a1dfca0aaa584f6fd0064c7de +size 1246165357 diff --git a/experiments/think-d12-r11-run2/config.json b/experiments/think-d12-r11-run2/config.json new file mode 100644 index 0000000000000000000000000000000000000000..a9540c3aaf44e16cf6d8327a67f0d6b7d7c3bacd --- /dev/null +++ b/experiments/think-d12-r11-run2/config.json @@ -0,0 +1,56 @@ +{ + "schema_version": 1, + "stage": "base", + "experiment_id": "think-d12-r11-run2", + "dataset": { + "adapter": "parquet_shards", + "repo": "jbduran/think-dataset", + "revision": "main", + "validation_shard": 472, + "num_train_shards": 24, + "download_workers": 4 + }, + "tokenizer": { + "mode": "train", + "max_chars": 2000000000, + "doc_cap": 10000, + "vocab_size": 32768 + }, + "pretokenize": { + "enabled": true, + "slack": 1.03, + "val_tokens": 20971520, + "shard_tokens": 100000000, + "tokenizer_threads": 8 + }, + "training": { + "depth": 12, + "scaling_params": 110100912, + "target_param_data_ratio": 11.25, + "window_pattern": "L", + "device_batch_size": 16, + "total_batch_size": 524288, + "seed": 43, + "save_every": 500, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "sample_every": -1 + }, + "artifacts": { + "repo": "jbduran/think.nano" + }, + "wandb": { + "entity": "jbduran-thinkingmachinesncsu", + "project": "think.nano", + "name": "think-d12-r11.25-run2", + "group": "think-d12", + "tags": [ + "think-dataset", + "d12", + "ratio11.25" + ] + }, + "config_fingerprint": "7f54bf07c5352d97", + "artifact_path": "experiments/think-d12-r11-run2" +} diff --git a/experiments/think-d12-r11-run2/evals/core.json b/experiments/think-d12-r11-run2/evals/core.json new file mode 100644 index 0000000000000000000000000000000000000000..57b3d7328323f1a22a14afe7390062b96de7932e --- /dev/null +++ b/experiments/think-d12-r11-run2/evals/core.json @@ -0,0 +1,56 @@ +{ + "model": "base_model (step 2362)", + "step": 2362, + "bpb": {}, + "core_metric": 0.06687854185240662, + "core_results": { + "hellaswag_zeroshot": 0.274347722530365, + "jeopardy": 0.0004723665479104966, + "bigbench_qa_wikidata": 0.07002607733011246, + "arc_easy": 0.3055555522441864, + "arc_challenge": 0.21672354638576508, + "copa": 0.5299999713897705, + "commonsense_qa": 0.315315306186676, + "piqa": 0.5424374341964722, + "openbook_qa": 0.23800000548362732, + "lambada_openai": 0.19406171143054962, + "hellaswag": 0.2761402130126953, + "winograd": 0.5494505763053894, + "winogrande": 0.48382002115249634, + "bigbench_dyck_languages": 0.1120000034570694, + "agi_eval_lsat_ar": 0.25217390060424805, + "bigbench_cs_algorithms": 0.3810606002807617, + "bigbench_operators": 0.11428572237491608, + "bigbench_repeat_copy_logic": 0.0, + "squad": 0.014569535851478577, + "coqa": 0.08042089641094208, + "boolq": 0.5547400712966919, + "bigbench_language_identification": 0.24949999153614044 + }, + "centered_results": { + "hellaswag_zeroshot": 0.032463630040486656, + "jeopardy": 0.0004723665479104966, + "bigbench_qa_wikidata": 0.07002607733011246, + "arc_easy": 0.0740740696589152, + "arc_challenge": -0.0443686048189799, + "copa": 0.059999942779541016, + "commonsense_qa": 0.144144132733345, + "piqa": 0.08487486839294434, + "openbook_qa": -0.015999992688496906, + "lambada_openai": 0.19406171143054962, + "hellaswag": 0.034853617350260414, + "winograd": 0.09890115261077881, + "winogrande": -0.032359957695007324, + "bigbench_dyck_languages": 0.1120000034570694, + "agi_eval_lsat_ar": 0.06521737575531004, + "bigbench_cs_algorithms": 0.3810606002807617, + "bigbench_operators": 0.11428572237491608, + "bigbench_repeat_copy_logic": 0.0, + "squad": 0.014569535851478577, + "coqa": 0.08042089641094208, + "boolq": -0.17173665448238973, + "bigbench_language_identification": 0.17436742743249772 + }, + "conditioned_samples": [], + "unconditioned_samples": [] +} \ No newline at end of file diff --git a/experiments/think-d12-r11-run2/evals/samples.json b/experiments/think-d12-r11-run2/evals/samples.json new file mode 100644 index 0000000000000000000000000000000000000000..306690ba6c68d1f8408e0285e62ab890674610b0 --- /dev/null +++ b/experiments/think-d12-r11-run2/evals/samples.json @@ -0,0 +1,48 @@ +{ + "model": "base_model (step 2362)", + "step": 2362, + "bpb": {}, + "core_metric": null, + "core_results": null, + "centered_results": null, + "conditioned_samples": [ + { + "prompt": "The capital of France is", + "text": "<|bos|>The capital of France is the capital of the French nation, and the capital of the French nation, the" + }, + { + "prompt": "The chemical symbol of gold is", + "text": "<|bos|>The chemical symbol of gold is the same as that of the sun, and the same as that of the moon" + }, + { + "prompt": "If yesterday was Friday, then tomorrow will be", + "text": "<|bos|>If yesterday was Friday, then tomorrow will be Saturday, and the day of the week will be the day of the week." + }, + { + "prompt": "The opposite of hot is", + "text": "<|bos|>The opposite of hot is the same as cold. \n\nThe latter is the same as the former, and" + }, + { + "prompt": "The planets of the solar system are:", + "text": "<|bos|>The planets of the solar system are: \n\n1. The sun, the moon, and the planets. \n\n2." + }, + { + "prompt": "My favorite color is", + "text": "<|bos|>My favorite color is a red, and I am a little afraid of it. I have been in" + }, + { + "prompt": "If 5*x + 3 = 13, then x is", + "text": "<|bos|>If 5*x + 3 = 13, then x is the number of the days of the week, and the number of days of the" + } + ], + "unconditioned_samples": [ + "<|bos|>adjustments. A concern called industrial, they say, is the glory of the Modern Cabinet, and when not mentioned to history, he would be scarcely deserving to be thought a relation.\n\nMr. President, though different from Hume and Lange, we have been met by Dr. Blair and Dr. Hawke, who asked me whether they knew that his father-Ladies excepted, the young sir was the father-Loyalty of this nation. My object in making inquiries is not so much about the origin of the plant, which is the object of their art, as about the principles of the formation of that plant. This is known not", + "<|bos|> illness. \n\nPiscatoruddy, July, 1861.\n\nMrs. Crook's daughter actively enjoyed theulsion Rossetti. She, however, did not obtain it in these cases, and indeed, the illness was distressful in the extreme, for Mrs. Lerchy was a very sympathetic person. \n\nShe was quite determined to leave it as a sane man could do.\n\nFor the first time she told me. that she enjoyed it with at least a semester, and intimated how bold the experiment was. The regular exercise played by the firm incessant swinging (sometimes) with the ball dancer, was not a", + "<|bos|>withrepook and sheikhs have never forgetful Siamese readers. Yet in what they have done in stretching or frustrating how he himself accomplished it are apparent to every one. The \n\nSherman illustrated literature and mythology from the Kansas \n\nState Trials; so the lesson of the chapter from Kansas City might have been learned,-from the felicity of men who in his \"Honourable Survey \" have devoted more space to subsidiary and gigantic literature than any other industrial literature. The experiences of \n\nAlfred E. Rogers-the Garrick, with the pieces of Wall hotel at Portland, donning the dress design of Cyrus Box and the campaign of Darwin", + "<|bos|> Kendrician politics, upon his return home notice of the extent and value of the invention he was writing of, proceeded to discuss the problem of Conservative Reform and the difficulties in the path ofReformation, the conditions under which the forces of reaction and progress must take shape, and finally made the statement which is often of the highest authority: It was to be 30 years ago, and then I may say that, for the moment, I was disdainfully and loudly opposed to the change 30 years ago, and feeling most deeply the want lies in modesty, taking every chance to preserve my traditions and to reach my goal. It was", + "<|bos|>Harvard College Library \n\nBOUGHT WITH A INCOME FROM THE BEQUEST OF CHARLES SUMNER, LL. D. \n\nMember of Parliament concerning negroways\n\nOF Harvard College Library \n\nEGATE \u0448\u0435 \n\nMarch 14, 1880\n\nAN U \n\nWH \n\nWILLIAM TROW & SON.\n\nNOV 1929: \n\nSure of the rights of\n\n- Department of Cor \n\nvor \n\nVrol, High Contract\n\nGONE EENT ENL \n\ng Cz1 le lasting v anything upon\n\nPREFACE. \n\nTHE book of this history of events which occurred long since has appeared to me to be one of the greatest calamities of human history. A", + "<|bos|>masters, not of special character. Governor Wise was always very accessible, and had to be both very frequent and repeated. The exclusion of Mr. North, the War Commissioner, from the pulpit, was all that prevented him from gaining some compensation for what he had voluntarily done,-an event which free trade did not long postpone. \n\nPhotograph by fluids. \n\n\"Admiral Porter,\" \"Marguerite\n\nTaylor, tropical gardens, lingam est Octobris, sub-contemporaneous view of Civita \n\nCecenas, Crystal Mountains, Camp Devon, Chatzie, and The Queen Christina stand", + "<|bos|>Army Neglected for beleaguered Petersburg\n\nen Voodone Vortige Eveden Maiefe\n\nannes Nad\u00e9\n\nVrain 1 a Coro 3.5 5.35 Leave was granted \n\n6 15 8.00\n\nFISHERMAN'S RIVELE \n\nby way of effecting a march. It seemed to the wearied men that the fighting was over and the last enemy was advancing. \n\nIn aeffected mood for the first time, they gave gage to Thetis as es Sortige Eulomsdanellov hone Amiaconseins", + "<|bos|>orations. (1) Provided with a library sufficiently standard conferring constructively the-Laurelian and \n\nChantian streams, or the History of \n\nSouth America, and give all instruction in such a subject. Provided with regard to these works the requested attention. \n\n(c) Next to the Agricultural library shall come the Engineering, and a topographical description of the properties occupied by the Western equipment, and afford references for the prices of books in general use by the engineer.\n\nGENERAL NUMBERS. \n\nGENERAL NUMBERS. \n\n6 Various maps and charts illustrating the operations of the British equipped ships of war which fought and captures seventeen countries" + ] +} \ No newline at end of file diff --git a/experiments/think-d12-r11-run2/evals/val_bpb.json b/experiments/think-d12-r11-run2/evals/val_bpb.json new file mode 100644 index 0000000000000000000000000000000000000000..d5c35476793f6d86232b1da4bfc6ac9077100a95 --- /dev/null +++ b/experiments/think-d12-r11-run2/evals/val_bpb.json @@ -0,0 +1,12 @@ +{ + "model": "base_model (step 2362)", + "step": 2362, + "bpb": { + "val": 1.0513915163986667 + }, + "core_metric": null, + "core_results": null, + "centered_results": null, + "conditioned_samples": [], + "unconditioned_samples": [] +} \ No newline at end of file diff --git a/experiments/think-d12-r11-run2/run.json b/experiments/think-d12-r11-run2/run.json new file mode 100644 index 0000000000000000000000000000000000000000..27b9c4c9937f118ed07f0d9fbbe51ed601156fca --- /dev/null +++ b/experiments/think-d12-r11-run2/run.json @@ -0,0 +1,10 @@ +{ + "experiment_id": "think-d12-r11-run2", + "stage": "base", + "base_experiment_id": "think-d12-r11-run2", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "config_fingerprint": "7f54bf07c5352d97", + "wandb_run_id": "84e31c8e", + "created_at": 1782060945 +} diff --git a/experiments/think-d12-r11-run2/summary.json b/experiments/think-d12-r11-run2/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..4409cb7084170c3bb8aa32a889cbdfe5d0d733c0 --- /dev/null +++ b/experiments/think-d12-r11-run2/summary.json @@ -0,0 +1,93 @@ +{ + "experiment_id": "think-d12-r11-run2", + "stage": "base", + "base_experiment_id": "think-d12-r11-run2", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "dataset": "jbduran/think-dataset", + "dataset_revision": "main", + "step": 2362, + "depth": 12, + "target_param_data_ratio": 11.25, + "training_tokens": 1238368256, + "final_sampled_val_bpb": 1.101818437651454, + "minimum_sampled_val_bpb": 1.101818437651454, + "full_val_bpb": 1.0513915163986667, + "core_metric": 0.06687854185240662, + "centered_results": { + "hellaswag_zeroshot": 0.032463630040486656, + "jeopardy": 0.0004723665479104966, + "bigbench_qa_wikidata": 0.07002607733011246, + "arc_easy": 0.0740740696589152, + "arc_challenge": -0.0443686048189799, + "copa": 0.059999942779541016, + "commonsense_qa": 0.144144132733345, + "piqa": 0.08487486839294434, + "openbook_qa": -0.015999992688496906, + "lambada_openai": 0.19406171143054962, + "hellaswag": 0.034853617350260414, + "winograd": 0.09890115261077881, + "winogrande": -0.032359957695007324, + "bigbench_dyck_languages": 0.1120000034570694, + "agi_eval_lsat_ar": 0.06521737575531004, + "bigbench_cs_algorithms": 0.3810606002807617, + "bigbench_operators": 0.11428572237491608, + "bigbench_repeat_copy_logic": 0.0, + "squad": 0.014569535851478577, + "coqa": 0.08042089641094208, + "boolq": -0.17173665448238973, + "bigbench_language_identification": 0.17436742743249772 + }, + "conditioned_samples": [ + { + "prompt": "The capital of France is", + "text": "<|bos|>The capital of France is the capital of the French nation, and the capital of the French nation, the" + }, + { + "prompt": "The chemical symbol of gold is", + "text": "<|bos|>The chemical symbol of gold is the same as that of the sun, and the same as that of the moon" + }, + { + "prompt": "If yesterday was Friday, then tomorrow will be", + "text": "<|bos|>If yesterday was Friday, then tomorrow will be Saturday, and the day of the week will be the day of the week." + }, + { + "prompt": "The opposite of hot is", + "text": "<|bos|>The opposite of hot is the same as cold. \n\nThe latter is the same as the former, and" + }, + { + "prompt": "The planets of the solar system are:", + "text": "<|bos|>The planets of the solar system are: \n\n1. The sun, the moon, and the planets. \n\n2." + }, + { + "prompt": "My favorite color is", + "text": "<|bos|>My favorite color is a red, and I am a little afraid of it. I have been in" + }, + { + "prompt": "If 5*x + 3 = 13, then x is", + "text": "<|bos|>If 5*x + 3 = 13, then x is the number of the days of the week, and the number of days of the" + } + ], + "unconditioned_samples": [ + "<|bos|>adjustments. A concern called industrial, they say, is the glory of the Modern Cabinet, and when not mentioned to history, he would be scarcely deserving to be thought a relation.\n\nMr. President, though different from Hume and Lange, we have been met by Dr. Blair and Dr. Hawke, who asked me whether they knew that his father-Ladies excepted, the young sir was the father-Loyalty of this nation. My object in making inquiries is not so much about the origin of the plant, which is the object of their art, as about the principles of the formation of that plant. This is known not", + "<|bos|> illness. \n\nPiscatoruddy, July, 1861.\n\nMrs. Crook's daughter actively enjoyed theulsion Rossetti. She, however, did not obtain it in these cases, and indeed, the illness was distressful in the extreme, for Mrs. Lerchy was a very sympathetic person. \n\nShe was quite determined to leave it as a sane man could do.\n\nFor the first time she told me. that she enjoyed it with at least a semester, and intimated how bold the experiment was. The regular exercise played by the firm incessant swinging (sometimes) with the ball dancer, was not a", + "<|bos|>withrepook and sheikhs have never forgetful Siamese readers. Yet in what they have done in stretching or frustrating how he himself accomplished it are apparent to every one. The \n\nSherman illustrated literature and mythology from the Kansas \n\nState Trials; so the lesson of the chapter from Kansas City might have been learned,-from the felicity of men who in his \"Honourable Survey \" have devoted more space to subsidiary and gigantic literature than any other industrial literature. The experiences of \n\nAlfred E. Rogers-the Garrick, with the pieces of Wall hotel at Portland, donning the dress design of Cyrus Box and the campaign of Darwin", + "<|bos|> Kendrician politics, upon his return home notice of the extent and value of the invention he was writing of, proceeded to discuss the problem of Conservative Reform and the difficulties in the path ofReformation, the conditions under which the forces of reaction and progress must take shape, and finally made the statement which is often of the highest authority: It was to be 30 years ago, and then I may say that, for the moment, I was disdainfully and loudly opposed to the change 30 years ago, and feeling most deeply the want lies in modesty, taking every chance to preserve my traditions and to reach my goal. It was", + "<|bos|>Harvard College Library \n\nBOUGHT WITH A INCOME FROM THE BEQUEST OF CHARLES SUMNER, LL. D. \n\nMember of Parliament concerning negroways\n\nOF Harvard College Library \n\nEGATE \u0448\u0435 \n\nMarch 14, 1880\n\nAN U \n\nWH \n\nWILLIAM TROW & SON.\n\nNOV 1929: \n\nSure of the rights of\n\n- Department of Cor \n\nvor \n\nVrol, High Contract\n\nGONE EENT ENL \n\ng Cz1 le lasting v anything upon\n\nPREFACE. \n\nTHE book of this history of events which occurred long since has appeared to me to be one of the greatest calamities of human history. A", + "<|bos|>masters, not of special character. Governor Wise was always very accessible, and had to be both very frequent and repeated. The exclusion of Mr. North, the War Commissioner, from the pulpit, was all that prevented him from gaining some compensation for what he had voluntarily done,-an event which free trade did not long postpone. \n\nPhotograph by fluids. \n\n\"Admiral Porter,\" \"Marguerite\n\nTaylor, tropical gardens, lingam est Octobris, sub-contemporaneous view of Civita \n\nCecenas, Crystal Mountains, Camp Devon, Chatzie, and The Queen Christina stand", + "<|bos|>Army Neglected for beleaguered Petersburg\n\nen Voodone Vortige Eveden Maiefe\n\nannes Nad\u00e9\n\nVrain 1 a Coro 3.5 5.35 Leave was granted \n\n6 15 8.00\n\nFISHERMAN'S RIVELE \n\nby way of effecting a march. It seemed to the wearied men that the fighting was over and the last enemy was advancing. \n\nIn aeffected mood for the first time, they gave gage to Thetis as es Sortige Eulomsdanellov hone Amiaconseins", + "<|bos|>orations. (1) Provided with a library sufficiently standard conferring constructively the-Laurelian and \n\nChantian streams, or the History of \n\nSouth America, and give all instruction in such a subject. Provided with regard to these works the requested attention. \n\n(c) Next to the Agricultural library shall come the Engineering, and a topographical description of the properties occupied by the Western equipment, and afford references for the prices of books in general use by the engineer.\n\nGENERAL NUMBERS. \n\nGENERAL NUMBERS. \n\n6 Various maps and charts illustrating the operations of the British equipped ships of war which fought and captures seventeen countries" + ], + "training_time_seconds": 6222.97850894928, + "stage_training_flops": 1.0985538644638433e+18, + "inherited_parent_flops": 0.0, + "cumulative_pipeline_training_flops": 1.0985538644638433e+18, + "config_fingerprint": "7f54bf07c5352d97", + "git_commit_sha": "48cd5f516123fcf119c7a43a32dd1dfd5c7b8eaa", + "wandb_url": "https://wandb.ai/jbduran-thinkingmachinesncsu/think.nano/runs/84e31c8e", + "huggingface_url": "https://huggingface.co/jbduran/think.nano/tree/main/experiments/think-d12-r11-run2", + "dataset_fingerprint": "63a5e6be81591d82", + "tokenizer_fingerprint": "96dd8e502a849d59", + "unique_train_tokens": 1275519304, + "effective_epochs": 0.970873786164196 +} diff --git a/experiments/think-d12-r11-run2/tokenizer/experiment_tokenizer.json b/experiments/think-d12-r11-run2/tokenizer/experiment_tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..ffa040a4514d028ed7fad16e82bee2dccd95ac58 --- /dev/null +++ b/experiments/think-d12-r11-run2/tokenizer/experiment_tokenizer.json @@ -0,0 +1,18 @@ +{ + "experiment_id": "think-d12-r11-run2", + "dataset": { + "adapter": "parquet_shards", + "repo": "jbduran/think-dataset", + "revision": "main", + "validation_shard": 472, + "num_train_shards": 24, + "download_workers": 4 + }, + "tokenizer": { + "mode": "train", + "max_chars": 2000000000, + "doc_cap": 10000, + "vocab_size": 32768 + }, + "created_at": 1782060961 +} diff --git a/experiments/think-d12-r11-run2/tokenizer/token_bytes.pt b/experiments/think-d12-r11-run2/tokenizer/token_bytes.pt new file mode 100644 index 0000000000000000000000000000000000000000..01d1ec4aab9e8a7d205c3b3ffbeb8da0e9a62db1 --- /dev/null +++ b/experiments/think-d12-r11-run2/tokenizer/token_bytes.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:53db25a1f6969198e67c303e479e97fb49859c715061baec525471666a2f47e1 +size 132649 diff --git a/experiments/think-d12-r11-run2/tokenizer/tokenizer.pkl b/experiments/think-d12-r11-run2/tokenizer/tokenizer.pkl new file mode 100644 index 0000000000000000000000000000000000000000..a17bd392980021628053b95d6425fc556aad527a --- /dev/null +++ b/experiments/think-d12-r11-run2/tokenizer/tokenizer.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:42ea52179dcefb47e4a4bbca7c37f67c3373840a3681587b1c8fe188f00d3bf1 +size 404071 diff --git a/experiments/think-d12-r11-run3/base_checkpoints/meta_000500.json b/experiments/think-d12-r11-run3/base_checkpoints/meta_000500.json new file mode 100644 index 0000000000000000000000000000000000000000..3e67690938c8373307f1fe4863c4cd2befb1aa49 --- /dev/null +++ b/experiments/think-d12-r11-run3/base_checkpoints/meta_000500.json @@ -0,0 +1,138 @@ +{ + "step": 500, + "experiment_id": "think-d12-r11-run3", + "val_bpb": 1.3216701028053668, + "model_config": { + "sequence_len": 2048, + "vocab_size": 32768, + "n_layer": 12, + "n_head": 6, + "n_kv_head": 6, + "n_embd": 768, + "window_pattern": "L" + }, + "user_config": { + "run": "think-d12-r11.25-run3", + "wandb_run_id": "46b7f9c8", + "wandb_group": "think-d12", + "wandb_tags": "think-dataset,d12,ratio11.25", + "device_type": "", + "fp8": false, + "fp8_recipe": "tensorwise", + "depth": 12, + "aspect_ratio": 64, + "head_dim": 128, + "max_seq_len": 2048, + "window_pattern": "L", + "num_iterations": -1, + "target_flops": -1.0, + "target_param_data_ratio": 11.25, + "device_batch_size": 16, + "total_batch_size": 524288, + "embedding_lr": 0.3, + "unembedding_lr": 0.008, + "weight_decay": 0.28, + "matrix_lr": 0.02, + "scalar_lr": 0.5, + "warmup_steps": 40, + "warmdown_ratio": 0.65, + "final_lr_frac": 0.05, + "resume_from_step": -1, + "pretokenized": true, + "data_dir": "/content/nanochat_cache/experiments/think-d12-r11-run3/data", + "tokenizer_dir": "/content/nanochat_cache/experiments/think-d12-r11-run3/tokenizer", + "pretokenized_dir": "/content/nanochat_cache/experiments/think-d12-r11-run3/pretok", + "checkpoint_dir": "/content/nanochat_cache/experiments/think-d12-r11-run3/base_checkpoints", + "experiment_id": "think-d12-r11-run3", + "experiment_config": "/content/nanochat_cache/experiments/think-d12-r11-run3/config.json", + "tokenizer_fingerprint": "85b2a26d1c355860", + "git_commit_sha": "48cd5f516123fcf119c7a43a32dd1dfd5c7b8eaa", + "seed": 44, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "core_metric_max_per_task": 500, + "sample_every": -1, + "save_every": 500, + "model_tag": "think-d12-r11-run3", + "experiment": { + "schema_version": 1, + "stage": "base", + "experiment_id": "think-d12-r11-run3", + "dataset": { + "adapter": "parquet_shards", + "repo": "jbduran/think-dataset", + "revision": "main", + "validation_shard": 472, + "num_train_shards": 24, + "download_workers": 4 + }, + "tokenizer": { + "mode": "train", + "max_chars": 2000000000, + "doc_cap": 10000, + "vocab_size": 32768 + }, + "pretokenize": { + "enabled": true, + "slack": 1.03, + "val_tokens": 20971520, + "shard_tokens": 100000000, + "tokenizer_threads": 8 + }, + "training": { + "depth": 12, + "scaling_params": 110100912, + "target_param_data_ratio": 11.25, + "window_pattern": "L", + "device_batch_size": 16, + "total_batch_size": 524288, + "seed": 44, + "save_every": 500, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "sample_every": -1 + }, + "artifacts": { + "repo": "jbduran/think.nano" + }, + "wandb": { + "entity": "jbduran-thinkingmachinesncsu", + "project": "think.nano", + "name": "think-d12-r11.25-run3", + "group": "think-d12", + "tags": [ + "think-dataset", + "d12", + "ratio11.25" + ] + }, + "config_fingerprint": "dd64eb8e857c4d1a", + "artifact_path": "experiments/think-d12-r11-run3" + }, + "stage": "base", + "base_experiment_id": "think-d12-r11-run3", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "config_fingerprint": "dd64eb8e857c4d1a" + }, + "device_batch_size": 16, + "max_seq_len": 2048, + "total_batch_size": 524288, + "dataloader_state_dict": { + "file_idx": 2, + "pos": 62184769, + "epoch": 1, + "pq_idx": 2, + "rg_idx": 62184769 + }, + "loop_state": { + "min_val_bpb": 1.3216701028053668, + "smooth_train_loss": 3.5803056875068, + "total_training_time": 1297.331782579422, + "stage_training_flops": 232547388751872000, + "inherited_parent_flops": 0.0, + "cumulative_pipeline_training_flops": 232547388751872000 + } +} \ No newline at end of file diff --git a/experiments/think-d12-r11-run3/base_checkpoints/meta_001000.json b/experiments/think-d12-r11-run3/base_checkpoints/meta_001000.json new file mode 100644 index 0000000000000000000000000000000000000000..9fd0ffd67fdee5b4ac0cd82e9e07d339caf255c3 --- /dev/null +++ b/experiments/think-d12-r11-run3/base_checkpoints/meta_001000.json @@ -0,0 +1,138 @@ +{ + "step": 1000, + "experiment_id": "think-d12-r11-run3", + "val_bpb": 1.2331196818438914, + "model_config": { + "sequence_len": 2048, + "vocab_size": 32768, + "n_layer": 12, + "n_head": 6, + "n_kv_head": 6, + "n_embd": 768, + "window_pattern": "L" + }, + "user_config": { + "run": "think-d12-r11.25-run3", + "wandb_run_id": "46b7f9c8", + "wandb_group": "think-d12", + "wandb_tags": "think-dataset,d12,ratio11.25", + "device_type": "", + "fp8": false, + "fp8_recipe": "tensorwise", + "depth": 12, + "aspect_ratio": 64, + "head_dim": 128, + "max_seq_len": 2048, + "window_pattern": "L", + "num_iterations": -1, + "target_flops": -1.0, + "target_param_data_ratio": 11.25, + "device_batch_size": 16, + "total_batch_size": 524288, + "embedding_lr": 0.3, + "unembedding_lr": 0.008, + "weight_decay": 0.28, + "matrix_lr": 0.02, + "scalar_lr": 0.5, + "warmup_steps": 40, + "warmdown_ratio": 0.65, + "final_lr_frac": 0.05, + "resume_from_step": -1, + "pretokenized": true, + "data_dir": "/content/nanochat_cache/experiments/think-d12-r11-run3/data", + "tokenizer_dir": "/content/nanochat_cache/experiments/think-d12-r11-run3/tokenizer", + "pretokenized_dir": "/content/nanochat_cache/experiments/think-d12-r11-run3/pretok", + "checkpoint_dir": "/content/nanochat_cache/experiments/think-d12-r11-run3/base_checkpoints", + "experiment_id": "think-d12-r11-run3", + "experiment_config": "/content/nanochat_cache/experiments/think-d12-r11-run3/config.json", + "tokenizer_fingerprint": "85b2a26d1c355860", + "git_commit_sha": "48cd5f516123fcf119c7a43a32dd1dfd5c7b8eaa", + "seed": 44, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "core_metric_max_per_task": 500, + "sample_every": -1, + "save_every": 500, + "model_tag": "think-d12-r11-run3", + "experiment": { + "schema_version": 1, + "stage": "base", + "experiment_id": "think-d12-r11-run3", + "dataset": { + "adapter": "parquet_shards", + "repo": "jbduran/think-dataset", + "revision": "main", + "validation_shard": 472, + "num_train_shards": 24, + "download_workers": 4 + }, + "tokenizer": { + "mode": "train", + "max_chars": 2000000000, + "doc_cap": 10000, + "vocab_size": 32768 + }, + "pretokenize": { + "enabled": true, + "slack": 1.03, + "val_tokens": 20971520, + "shard_tokens": 100000000, + "tokenizer_threads": 8 + }, + "training": { + "depth": 12, + "scaling_params": 110100912, + "target_param_data_ratio": 11.25, + "window_pattern": "L", + "device_batch_size": 16, + "total_batch_size": 524288, + "seed": 44, + "save_every": 500, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "sample_every": -1 + }, + "artifacts": { + "repo": "jbduran/think.nano" + }, + "wandb": { + "entity": "jbduran-thinkingmachinesncsu", + "project": "think.nano", + "name": "think-d12-r11.25-run3", + "group": "think-d12", + "tags": [ + "think-dataset", + "d12", + "ratio11.25" + ] + }, + "config_fingerprint": "dd64eb8e857c4d1a", + "artifact_path": "experiments/think-d12-r11-run3" + }, + "stage": "base", + "base_experiment_id": "think-d12-r11-run3", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "config_fingerprint": "dd64eb8e857c4d1a" + }, + "device_batch_size": 16, + "max_seq_len": 2048, + "total_batch_size": 524288, + "dataloader_state_dict": { + "file_idx": 5, + "pos": 24336769, + "epoch": 1, + "pq_idx": 5, + "rg_idx": 24336769 + }, + "loop_state": { + "min_val_bpb": 1.2331196818438914, + "smooth_train_loss": 3.4406616937694783, + "total_training_time": 2622.119250535965, + "stage_training_flops": 465094777503744000, + "inherited_parent_flops": 0.0, + "cumulative_pipeline_training_flops": 465094777503744000 + } +} \ No newline at end of file diff --git a/experiments/think-d12-r11-run3/base_checkpoints/meta_001500.json b/experiments/think-d12-r11-run3/base_checkpoints/meta_001500.json new file mode 100644 index 0000000000000000000000000000000000000000..69a7a36cb44c60b3ff0a5feb5e8a62088339f9ee --- /dev/null +++ b/experiments/think-d12-r11-run3/base_checkpoints/meta_001500.json @@ -0,0 +1,138 @@ +{ + "step": 1500, + "experiment_id": "think-d12-r11-run3", + "val_bpb": 1.174251658716475, + "model_config": { + "sequence_len": 2048, + "vocab_size": 32768, + "n_layer": 12, + "n_head": 6, + "n_kv_head": 6, + "n_embd": 768, + "window_pattern": "L" + }, + "user_config": { + "run": "think-d12-r11.25-run3", + "wandb_run_id": "46b7f9c8", + "wandb_group": "think-d12", + "wandb_tags": "think-dataset,d12,ratio11.25", + "device_type": "", + "fp8": false, + "fp8_recipe": "tensorwise", + "depth": 12, + "aspect_ratio": 64, + "head_dim": 128, + "max_seq_len": 2048, + "window_pattern": "L", + "num_iterations": -1, + "target_flops": -1.0, + "target_param_data_ratio": 11.25, + "device_batch_size": 16, + "total_batch_size": 524288, + "embedding_lr": 0.3, + "unembedding_lr": 0.008, + "weight_decay": 0.28, + "matrix_lr": 0.02, + "scalar_lr": 0.5, + "warmup_steps": 40, + "warmdown_ratio": 0.65, + "final_lr_frac": 0.05, + "resume_from_step": 1000, + "pretokenized": true, + "data_dir": "/content/nanochat_cache/experiments/think-d12-r11-run3/data", + "tokenizer_dir": "/content/nanochat_cache/experiments/think-d12-r11-run3/tokenizer", + "pretokenized_dir": "/content/nanochat_cache/experiments/think-d12-r11-run3/pretok", + "checkpoint_dir": "/content/nanochat_cache/experiments/think-d12-r11-run3/base_checkpoints", + "experiment_id": "think-d12-r11-run3", + "experiment_config": "/content/nanochat_cache/experiments/think-d12-r11-run3/config.json", + "tokenizer_fingerprint": "85b2a26d1c355860", + "git_commit_sha": "48cd5f516123fcf119c7a43a32dd1dfd5c7b8eaa", + "seed": 44, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "core_metric_max_per_task": 500, + "sample_every": -1, + "save_every": 500, + "model_tag": "think-d12-r11-run3", + "experiment": { + "schema_version": 1, + "stage": "base", + "experiment_id": "think-d12-r11-run3", + "dataset": { + "adapter": "parquet_shards", + "repo": "jbduran/think-dataset", + "revision": "main", + "validation_shard": 472, + "num_train_shards": 24, + "download_workers": 4 + }, + "tokenizer": { + "mode": "train", + "max_chars": 2000000000, + "doc_cap": 10000, + "vocab_size": 32768 + }, + "pretokenize": { + "enabled": true, + "slack": 1.03, + "val_tokens": 20971520, + "shard_tokens": 100000000, + "tokenizer_threads": 8 + }, + "training": { + "depth": 12, + "scaling_params": 110100912, + "target_param_data_ratio": 11.25, + "window_pattern": "L", + "device_batch_size": 16, + "total_batch_size": 524288, + "seed": 44, + "save_every": 500, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "sample_every": -1 + }, + "artifacts": { + "repo": "jbduran/think.nano" + }, + "wandb": { + "entity": "jbduran-thinkingmachinesncsu", + "project": "think.nano", + "name": "think-d12-r11.25-run3", + "group": "think-d12", + "tags": [ + "think-dataset", + "d12", + "ratio11.25" + ] + }, + "config_fingerprint": "dd64eb8e857c4d1a", + "artifact_path": "experiments/think-d12-r11-run3" + }, + "stage": "base", + "base_experiment_id": "think-d12-r11-run3", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "config_fingerprint": "dd64eb8e857c4d1a" + }, + "device_batch_size": 16, + "max_seq_len": 2048, + "total_batch_size": 524288, + "dataloader_state_dict": { + "file_idx": 7, + "pos": 86521538, + "epoch": 1, + "pq_idx": 7, + "rg_idx": 86521538 + }, + "loop_state": { + "min_val_bpb": 1.174251658716475, + "smooth_train_loss": 3.270629875998561, + "total_training_time": 4003.970594406128, + "stage_training_flops": 697642166255616000, + "inherited_parent_flops": 0.0, + "cumulative_pipeline_training_flops": 697642166255616000 + } +} \ No newline at end of file diff --git a/experiments/think-d12-r11-run3/base_checkpoints/meta_002000.json b/experiments/think-d12-r11-run3/base_checkpoints/meta_002000.json new file mode 100644 index 0000000000000000000000000000000000000000..67da3c8a2070e13b22a251123219ea99709b4d4a --- /dev/null +++ b/experiments/think-d12-r11-run3/base_checkpoints/meta_002000.json @@ -0,0 +1,138 @@ +{ + "step": 2000, + "experiment_id": "think-d12-r11-run3", + "val_bpb": 1.1220845787641187, + "model_config": { + "sequence_len": 2048, + "vocab_size": 32768, + "n_layer": 12, + "n_head": 6, + "n_kv_head": 6, + "n_embd": 768, + "window_pattern": "L" + }, + "user_config": { + "run": "think-d12-r11.25-run3", + "wandb_run_id": "46b7f9c8", + "wandb_group": "think-d12", + "wandb_tags": "think-dataset,d12,ratio11.25", + "device_type": "", + "fp8": false, + "fp8_recipe": "tensorwise", + "depth": 12, + "aspect_ratio": 64, + "head_dim": 128, + "max_seq_len": 2048, + "window_pattern": "L", + "num_iterations": -1, + "target_flops": -1.0, + "target_param_data_ratio": 11.25, + "device_batch_size": 16, + "total_batch_size": 524288, + "embedding_lr": 0.3, + "unembedding_lr": 0.008, + "weight_decay": 0.28, + "matrix_lr": 0.02, + "scalar_lr": 0.5, + "warmup_steps": 40, + "warmdown_ratio": 0.65, + "final_lr_frac": 0.05, + "resume_from_step": 1000, + "pretokenized": true, + "data_dir": "/content/nanochat_cache/experiments/think-d12-r11-run3/data", + "tokenizer_dir": "/content/nanochat_cache/experiments/think-d12-r11-run3/tokenizer", + "pretokenized_dir": "/content/nanochat_cache/experiments/think-d12-r11-run3/pretok", + "checkpoint_dir": "/content/nanochat_cache/experiments/think-d12-r11-run3/base_checkpoints", + "experiment_id": "think-d12-r11-run3", + "experiment_config": "/content/nanochat_cache/experiments/think-d12-r11-run3/config.json", + "tokenizer_fingerprint": "85b2a26d1c355860", + "git_commit_sha": "48cd5f516123fcf119c7a43a32dd1dfd5c7b8eaa", + "seed": 44, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "core_metric_max_per_task": 500, + "sample_every": -1, + "save_every": 500, + "model_tag": "think-d12-r11-run3", + "experiment": { + "schema_version": 1, + "stage": "base", + "experiment_id": "think-d12-r11-run3", + "dataset": { + "adapter": "parquet_shards", + "repo": "jbduran/think-dataset", + "revision": "main", + "validation_shard": 472, + "num_train_shards": 24, + "download_workers": 4 + }, + "tokenizer": { + "mode": "train", + "max_chars": 2000000000, + "doc_cap": 10000, + "vocab_size": 32768 + }, + "pretokenize": { + "enabled": true, + "slack": 1.03, + "val_tokens": 20971520, + "shard_tokens": 100000000, + "tokenizer_threads": 8 + }, + "training": { + "depth": 12, + "scaling_params": 110100912, + "target_param_data_ratio": 11.25, + "window_pattern": "L", + "device_batch_size": 16, + "total_batch_size": 524288, + "seed": 44, + "save_every": 500, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "sample_every": -1 + }, + "artifacts": { + "repo": "jbduran/think.nano" + }, + "wandb": { + "entity": "jbduran-thinkingmachinesncsu", + "project": "think.nano", + "name": "think-d12-r11.25-run3", + "group": "think-d12", + "tags": [ + "think-dataset", + "d12", + "ratio11.25" + ] + }, + "config_fingerprint": "dd64eb8e857c4d1a", + "artifact_path": "experiments/think-d12-r11-run3" + }, + "stage": "base", + "base_experiment_id": "think-d12-r11-run3", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "config_fingerprint": "dd64eb8e857c4d1a" + }, + "device_batch_size": 16, + "max_seq_len": 2048, + "total_batch_size": 524288, + "dataloader_state_dict": { + "file_idx": 10, + "pos": 48673538, + "epoch": 1, + "pq_idx": 10, + "rg_idx": 48673538 + }, + "loop_state": { + "min_val_bpb": 1.1220845787641187, + "smooth_train_loss": 3.242369223248623, + "total_training_time": 5326.6880078315735, + "stage_training_flops": 930189555007488000, + "inherited_parent_flops": 0.0, + "cumulative_pipeline_training_flops": 930189555007488000 + } +} \ No newline at end of file diff --git a/experiments/think-d12-r11-run3/base_checkpoints/meta_002362.json b/experiments/think-d12-r11-run3/base_checkpoints/meta_002362.json new file mode 100644 index 0000000000000000000000000000000000000000..724c3b3c081bebb8276ada2c519c53d38a81e343 --- /dev/null +++ b/experiments/think-d12-r11-run3/base_checkpoints/meta_002362.json @@ -0,0 +1,138 @@ +{ + "step": 2362, + "experiment_id": "think-d12-r11-run3", + "val_bpb": 1.1016183928831433, + "model_config": { + "sequence_len": 2048, + "vocab_size": 32768, + "n_layer": 12, + "n_head": 6, + "n_kv_head": 6, + "n_embd": 768, + "window_pattern": "L" + }, + "user_config": { + "run": "think-d12-r11.25-run3", + "wandb_run_id": "46b7f9c8", + "wandb_group": "think-d12", + "wandb_tags": "think-dataset,d12,ratio11.25", + "device_type": "", + "fp8": false, + "fp8_recipe": "tensorwise", + "depth": 12, + "aspect_ratio": 64, + "head_dim": 128, + "max_seq_len": 2048, + "window_pattern": "L", + "num_iterations": -1, + "target_flops": -1.0, + "target_param_data_ratio": 11.25, + "device_batch_size": 16, + "total_batch_size": 524288, + "embedding_lr": 0.3, + "unembedding_lr": 0.008, + "weight_decay": 0.28, + "matrix_lr": 0.02, + "scalar_lr": 0.5, + "warmup_steps": 40, + "warmdown_ratio": 0.65, + "final_lr_frac": 0.05, + "resume_from_step": 1000, + "pretokenized": true, + "data_dir": "/content/nanochat_cache/experiments/think-d12-r11-run3/data", + "tokenizer_dir": "/content/nanochat_cache/experiments/think-d12-r11-run3/tokenizer", + "pretokenized_dir": "/content/nanochat_cache/experiments/think-d12-r11-run3/pretok", + "checkpoint_dir": "/content/nanochat_cache/experiments/think-d12-r11-run3/base_checkpoints", + "experiment_id": "think-d12-r11-run3", + "experiment_config": "/content/nanochat_cache/experiments/think-d12-r11-run3/config.json", + "tokenizer_fingerprint": "85b2a26d1c355860", + "git_commit_sha": "48cd5f516123fcf119c7a43a32dd1dfd5c7b8eaa", + "seed": 44, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "core_metric_max_per_task": 500, + "sample_every": -1, + "save_every": 500, + "model_tag": "think-d12-r11-run3", + "experiment": { + "schema_version": 1, + "stage": "base", + "experiment_id": "think-d12-r11-run3", + "dataset": { + "adapter": "parquet_shards", + "repo": "jbduran/think-dataset", + "revision": "main", + "validation_shard": 472, + "num_train_shards": 24, + "download_workers": 4 + }, + "tokenizer": { + "mode": "train", + "max_chars": 2000000000, + "doc_cap": 10000, + "vocab_size": 32768 + }, + "pretokenize": { + "enabled": true, + "slack": 1.03, + "val_tokens": 20971520, + "shard_tokens": 100000000, + "tokenizer_threads": 8 + }, + "training": { + "depth": 12, + "scaling_params": 110100912, + "target_param_data_ratio": 11.25, + "window_pattern": "L", + "device_batch_size": 16, + "total_batch_size": 524288, + "seed": 44, + "save_every": 500, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "sample_every": -1 + }, + "artifacts": { + "repo": "jbduran/think.nano" + }, + "wandb": { + "entity": "jbduran-thinkingmachinesncsu", + "project": "think.nano", + "name": "think-d12-r11.25-run3", + "group": "think-d12", + "tags": [ + "think-dataset", + "d12", + "ratio11.25" + ] + }, + "config_fingerprint": "dd64eb8e857c4d1a", + "artifact_path": "experiments/think-d12-r11-run3" + }, + "stage": "base", + "base_experiment_id": "think-d12-r11-run3", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "config_fingerprint": "dd64eb8e857c4d1a" + }, + "device_batch_size": 16, + "max_seq_len": 2048, + "total_batch_size": 524288, + "dataloader_state_dict": { + "file_idx": 12, + "pos": 38471586, + "epoch": 1, + "pq_idx": 12, + "rg_idx": 38471586 + }, + "loop_state": { + "min_val_bpb": 1.1016183928831433, + "smooth_train_loss": 3.111287829182217, + "total_training_time": 6288.5381960868835, + "stage_training_flops": 1098553864463843328, + "inherited_parent_flops": 0.0, + "cumulative_pipeline_training_flops": 1098553864463843328 + } +} \ No newline at end of file diff --git a/experiments/think-d12-r11-run3/base_checkpoints/model_000500.pt b/experiments/think-d12-r11-run3/base_checkpoints/model_000500.pt new file mode 100644 index 0000000000000000000000000000000000000000..a7b51a4315d7837ddd8ca931364fefd2d9ea9157 --- /dev/null +++ b/experiments/think-d12-r11-run3/base_checkpoints/model_000500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:281b1d08b67e20c8316754beb155a25f66736a2bb1bc705c782e792b9c5b2e1e +size 792761690 diff --git a/experiments/think-d12-r11-run3/base_checkpoints/model_001000.pt b/experiments/think-d12-r11-run3/base_checkpoints/model_001000.pt new file mode 100644 index 0000000000000000000000000000000000000000..8136e7c89e12bc0f4759d5a5b9188dd3aba225bb --- /dev/null +++ b/experiments/think-d12-r11-run3/base_checkpoints/model_001000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:36c5a9c879666bacd2eff38a8cefd065891833d2d3dfed5b40464c3b71897406 +size 792761690 diff --git a/experiments/think-d12-r11-run3/base_checkpoints/model_001500.pt b/experiments/think-d12-r11-run3/base_checkpoints/model_001500.pt new file mode 100644 index 0000000000000000000000000000000000000000..7b46061e2a0cefe3167b8eda8056d6c171952351 --- /dev/null +++ b/experiments/think-d12-r11-run3/base_checkpoints/model_001500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aaf3c2a3d0451c939f0742a0faff6f246ae25d3b9cebbc62fa6d9e4e27dddac7 +size 792761690 diff --git a/experiments/think-d12-r11-run3/base_checkpoints/model_002000.pt b/experiments/think-d12-r11-run3/base_checkpoints/model_002000.pt new file mode 100644 index 0000000000000000000000000000000000000000..32ef78ea87571e6d11dbaca89063341132ad1d6e --- /dev/null +++ b/experiments/think-d12-r11-run3/base_checkpoints/model_002000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:67ebd7cd1447180f6c465b5f7e554f06dbe71855c2b601f3b72a1ac86e759ef4 +size 792761690 diff --git a/experiments/think-d12-r11-run3/base_checkpoints/model_002362.pt b/experiments/think-d12-r11-run3/base_checkpoints/model_002362.pt new file mode 100644 index 0000000000000000000000000000000000000000..7ba71b24594223af13a491842e9fa1847b7058ff --- /dev/null +++ b/experiments/think-d12-r11-run3/base_checkpoints/model_002362.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fde26034120b061feb782e40d54d7efec1a8da9bda92f185eb9345749f89db7f +size 792761690 diff --git a/experiments/think-d12-r11-run3/base_checkpoints/optim_000500_rank0.pt b/experiments/think-d12-r11-run3/base_checkpoints/optim_000500_rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..73d97421f2215f2fff8f3748e3fef557f5fd3351 --- /dev/null +++ b/experiments/think-d12-r11-run3/base_checkpoints/optim_000500_rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1b95b914154f1ad2794f0a3aea4ff377d76072ac0c41d442e82db7aea59817e9 +size 1246165357 diff --git a/experiments/think-d12-r11-run3/base_checkpoints/optim_001000_rank0.pt b/experiments/think-d12-r11-run3/base_checkpoints/optim_001000_rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..11ba067553e261025de6af70769bab07d7880495 --- /dev/null +++ b/experiments/think-d12-r11-run3/base_checkpoints/optim_001000_rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d004ede0d800d74ffa6f5f0540afeeba530eb86630bd95c07d88469223365f07 +size 1246165357 diff --git a/experiments/think-d12-r11-run3/base_checkpoints/optim_001500_rank0.pt b/experiments/think-d12-r11-run3/base_checkpoints/optim_001500_rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..0d2631c42bfb103a13cc0c317f69ca53482b13f3 --- /dev/null +++ b/experiments/think-d12-r11-run3/base_checkpoints/optim_001500_rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2de50aba55cc93977007dd0dadcd69dfbf7ccc7ee2709b2fafcb28607714538b +size 1246165357 diff --git a/experiments/think-d12-r11-run3/base_checkpoints/optim_002000_rank0.pt b/experiments/think-d12-r11-run3/base_checkpoints/optim_002000_rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..27b6ed0348bd88bc560cde4e36079b7b4b514d1e --- /dev/null +++ b/experiments/think-d12-r11-run3/base_checkpoints/optim_002000_rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c43386f333717ad577befe466f6aa79408a866500e9a09ff3843f4ac7c484586 +size 1246165357 diff --git a/experiments/think-d12-r11-run3/base_checkpoints/optim_002362_rank0.pt b/experiments/think-d12-r11-run3/base_checkpoints/optim_002362_rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..5c0094ba79390a34776ebe69c83e03d187faa447 --- /dev/null +++ b/experiments/think-d12-r11-run3/base_checkpoints/optim_002362_rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fee73bb716f0dcb81bce2ab65dc7842501e7de6da53f20db7f815f8156749d98 +size 1246165357 diff --git a/experiments/think-d12-r11-run3/config.json b/experiments/think-d12-r11-run3/config.json new file mode 100644 index 0000000000000000000000000000000000000000..b7ccf5395ba44dcd753f1118d2524a1eb7decb9a --- /dev/null +++ b/experiments/think-d12-r11-run3/config.json @@ -0,0 +1,56 @@ +{ + "schema_version": 1, + "stage": "base", + "experiment_id": "think-d12-r11-run3", + "dataset": { + "adapter": "parquet_shards", + "repo": "jbduran/think-dataset", + "revision": "main", + "validation_shard": 472, + "num_train_shards": 24, + "download_workers": 4 + }, + "tokenizer": { + "mode": "train", + "max_chars": 2000000000, + "doc_cap": 10000, + "vocab_size": 32768 + }, + "pretokenize": { + "enabled": true, + "slack": 1.03, + "val_tokens": 20971520, + "shard_tokens": 100000000, + "tokenizer_threads": 8 + }, + "training": { + "depth": 12, + "scaling_params": 110100912, + "target_param_data_ratio": 11.25, + "window_pattern": "L", + "device_batch_size": 16, + "total_batch_size": 524288, + "seed": 44, + "save_every": 500, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "sample_every": -1 + }, + "artifacts": { + "repo": "jbduran/think.nano" + }, + "wandb": { + "entity": "jbduran-thinkingmachinesncsu", + "project": "think.nano", + "name": "think-d12-r11.25-run3", + "group": "think-d12", + "tags": [ + "think-dataset", + "d12", + "ratio11.25" + ] + }, + "config_fingerprint": "dd64eb8e857c4d1a", + "artifact_path": "experiments/think-d12-r11-run3" +} diff --git a/experiments/think-d12-r11-run3/evals/core.json b/experiments/think-d12-r11-run3/evals/core.json new file mode 100644 index 0000000000000000000000000000000000000000..60c6a59bd0e6a5b4da21d8397c699a381889cc45 --- /dev/null +++ b/experiments/think-d12-r11-run3/evals/core.json @@ -0,0 +1,56 @@ +{ + "model": "base_model (step 2362)", + "step": 2362, + "bpb": {}, + "core_metric": 0.06826829817538614, + "core_results": { + "hellaswag_zeroshot": 0.2783310115337372, + "jeopardy": 0.0004723665479104966, + "bigbench_qa_wikidata": 0.07568524777889252, + "arc_easy": 0.3097642958164215, + "arc_challenge": 0.20307166874408722, + "copa": 0.5099999904632568, + "commonsense_qa": 0.2907452881336212, + "piqa": 0.5435255765914917, + "openbook_qa": 0.24800001084804535, + "lambada_openai": 0.23073936998844147, + "hellaswag": 0.27653852105140686, + "winograd": 0.5384615659713745, + "winogrande": 0.5146014094352722, + "bigbench_dyck_languages": 0.08300000429153442, + "agi_eval_lsat_ar": 0.2695651948451996, + "bigbench_cs_algorithms": 0.42424240708351135, + "bigbench_operators": 0.07619047909975052, + "bigbench_repeat_copy_logic": 0.0, + "squad": 0.01551560964435339, + "coqa": 0.06776900589466095, + "boolq": 0.5620794892311096, + "bigbench_language_identification": 0.2541999816894531 + }, + "centered_results": { + "hellaswag_zeroshot": 0.03777468204498291, + "jeopardy": 0.0004723665479104966, + "bigbench_qa_wikidata": 0.07568524777889252, + "arc_easy": 0.07968572775522868, + "arc_challenge": -0.06257110834121704, + "copa": 0.019999980926513672, + "commonsense_qa": 0.1134316101670265, + "piqa": 0.0870511531829834, + "openbook_qa": -0.002666652202606201, + "lambada_openai": 0.23073936998844147, + "hellaswag": 0.035384694735209145, + "winograd": 0.07692313194274902, + "winogrande": 0.029202818870544434, + "bigbench_dyck_languages": 0.08300000429153442, + "agi_eval_lsat_ar": 0.08695649355649947, + "bigbench_cs_algorithms": 0.42424240708351135, + "bigbench_operators": 0.07619047909975052, + "bigbench_repeat_copy_logic": 0.0, + "squad": 0.01551560964435339, + "coqa": 0.06776900589466095, + "boolq": -0.15242239676023783, + "bigbench_language_identification": 0.1795379336517636 + }, + "conditioned_samples": [], + "unconditioned_samples": [] +} \ No newline at end of file diff --git a/experiments/think-d12-r11-run3/evals/samples.json b/experiments/think-d12-r11-run3/evals/samples.json new file mode 100644 index 0000000000000000000000000000000000000000..ad74ff0fe81dda9009004e4c22819bf878f5f5eb --- /dev/null +++ b/experiments/think-d12-r11-run3/evals/samples.json @@ -0,0 +1,48 @@ +{ + "model": "base_model (step 2362)", + "step": 2362, + "bpb": {}, + "core_metric": null, + "core_results": null, + "centered_results": null, + "conditioned_samples": [ + { + "prompt": "The capital of France is", + "text": "<|bos|>The capital of France is the capital of the world. \n\nThe capital of the world is the capital of" + }, + { + "prompt": "The chemical symbol of gold is", + "text": "<|bos|>The chemical symbol of gold is a gold chain, and the symbol of the gold chain is a gold chain." + }, + { + "prompt": "If yesterday was Friday, then tomorrow will be", + "text": "<|bos|>If yesterday was Friday, then tomorrow will be Saturday. \n\nI am not sure that I shall be able to go to-morrow" + }, + { + "prompt": "The opposite of hot is", + "text": "<|bos|>The opposite of hot is the opposite of cold. \n\nThe former is the more intense, the latter the" + }, + { + "prompt": "The planets of the solar system are:", + "text": "<|bos|>The planets of the solar system are: \n\n1. The sun, the moon, the stars, the sun, the" + }, + { + "prompt": "My favorite color is", + "text": "<|bos|>My favorite color is a blue, and I have a blue color. \n\nI have a blue color" + }, + { + "prompt": "If 5*x + 3 = 13, then x is", + "text": "<|bos|>If 5*x + 3 = 13, then x is 13, and 13 = 13, and 13 = 13" + } + ], + "unconditioned_samples": [ + "<|bos|>AL PSALTER PSALTER SCRIPTURE PSALTER CHR CHILDREN \n\nChristenome for Modern Sinners Psamates Sallarememorative Psalms Biblioteca \n\nDalmatique Italiano\n\nEaster exegetica Elisha Scotti Netherlandsnio Italiano \n\nDante Bert Raphael Exposition Cant Cant Cant Cant Cant Cant Cant Cant cant\n\nBible Jewish D-Lo spelling Cant Cant \n\nInitiatory Levant Horne\n\nItaliano-Anglo Jupe\n\nFolklore Spanish Olino Florinda In festo \n\nItaliano-Romano Guglielmoi \n\nGui Ser vocali Ministeri Burton", + "<|bos|> salary of the past year, including French and German assistance. The prospects of the business are very gloomy in this country. Rossetti is living, as a manager, in perfect peace. Often, it is said, he sets his empty purse at 73 per cent. It is probable that a very large amount of this is withdrawn every year. But it is probable also that the experience of the last two years is very different. He has been connected with at least one newspaper, having never been in any enterprise except the one that regular news service requires. One incessant fight he made with an old woman who was drugging and disguising herself", + "<|bos|> us Free men and she who could never excuse forgetful Siamese. Well, in the last grand war such men had nothing to lose. Thank you, for your rain, and say 'Good-bye.' I'll all this long. \n\nWith kind Lady Cassoit. \n\nYours very affectionately, \n\nSQUIRE SHANGHANK. \n\nL. SHERgee.\n\n'You were got so devotedly tied up by feeling to this morning that you struggled against it long with all your might when it was taken from me. But, as soon as you had won your way to your old friends and dear home, then I had", + "<|bos|> Kendenthal, Brig.-Gen. \n\nWaller, Mutiny Washington, 9 U.S. 451.\n\nMaryland loss at Manila.-Send prompt orders in case there is any error, the business is completed in twenty days.\n\nAmerican forces at Geba.---Actinon.-S. of the enemy at Geba. C.B. from active duties, natives, tude; valor (Georgia men in hospital), and privates, militia, 3,000 tons War Department. removed shipping lies to the colonies taking contraband of war, removing twelve hundred stone from one stockading", + "<|bos|> HOUSE OF THE LION RIOTS. \n\nAgate.] One who is anxious to get a glimpse into the future of My Majesty in this world, and out of the midst of error, far from denying himself marriage, treats it one who has hitherto trusted himself to carry on the war with difficulty and success: unincorporated, not, as some think, out of discouragement, but because he has shown himself so willing to enter on higher ground; often, not, perhaps, without some little reflection for his own sake, but without question after long and fatiguing deliberation; but, when earnest, unwearied and useful labour rises up", + "<|bos|>1690, 1 August. Governor Nathaniel Claffel. \n\nVan Isle.\n\nU. S. Army. \n\n1 June. General Cooper. \n\n1 June. Colonel Theodore Astley. \n\n1 July. Commander Joseph Palmer.\n\nEntom., Joshua Billings, John Hill, \n\n8 June. Captain John Daniels \n\nPhoebe Talbotton \n\n\"Admiral Porter,\" \"Marguerite\n\nNewburne \n\ntwo or three names for Oct. 14th. \n\nBertie Mills\n\nEmery Park\n\nLaugwanton \n\nCapt. William Ludlow \n\nHezekiah Meyers \n\nMosby", + "<|bos|>Army in the field and defends itself by an effective briinjade. He did not resort to threats, nor to use a poniard; but he only levied dreadfully upon the French who were marching up and down the Army.\n\n\"How should it be practicable now?\" said theLOWERS, while the Army was effecting its march. \"The army should be kept intact. The furnishing of the necessary subsistence should be strictly enforced. Thee should have these river-banks filled up, and garrisons The Executive has orders to inflict extreme corrup-\n\nviction soon on the enemy, because they are", + "<|bos|>the eightieth year of his age.\n\nI had never been conferring, like the paleface VIII. of the Conquests, on the History of \n\nSouth America, and give all the name ever given this island to Penrith.' \n\nLectures to the Greeks. \n\n[See preceding list.] 119-113, and a vaulted great hall.- \n\nApart from the three triumphs above described, there are three books in the Museum that are universally known to the present generation of students. The last has the date \n\n1184.'-Journal of the Institute of Arts, Sciences and Arts, No. 725" + ] +} \ No newline at end of file diff --git a/experiments/think-d12-r11-run3/evals/val_bpb.json b/experiments/think-d12-r11-run3/evals/val_bpb.json new file mode 100644 index 0000000000000000000000000000000000000000..0f4a6908a0bafe7e43f38a10d9237eaf5ef603f4 --- /dev/null +++ b/experiments/think-d12-r11-run3/evals/val_bpb.json @@ -0,0 +1,12 @@ +{ + "model": "base_model (step 2362)", + "step": 2362, + "bpb": { + "val": 1.0504735756248413 + }, + "core_metric": null, + "core_results": null, + "centered_results": null, + "conditioned_samples": [], + "unconditioned_samples": [] +} \ No newline at end of file diff --git a/experiments/think-d12-r11-run3/run.json b/experiments/think-d12-r11-run3/run.json new file mode 100644 index 0000000000000000000000000000000000000000..557535ce6a279f6a53b41e5a65e1e399c2e31f2c --- /dev/null +++ b/experiments/think-d12-r11-run3/run.json @@ -0,0 +1,10 @@ +{ + "experiment_id": "think-d12-r11-run3", + "stage": "base", + "base_experiment_id": "think-d12-r11-run3", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "config_fingerprint": "dd64eb8e857c4d1a", + "wandb_run_id": "46b7f9c8", + "created_at": 1782133395 +} diff --git a/experiments/think-d12-r11-run3/summary.json b/experiments/think-d12-r11-run3/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..5aaf8e216cbb8f830bc9573da6a414fd1883a760 --- /dev/null +++ b/experiments/think-d12-r11-run3/summary.json @@ -0,0 +1,93 @@ +{ + "experiment_id": "think-d12-r11-run3", + "stage": "base", + "base_experiment_id": "think-d12-r11-run3", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "dataset": "jbduran/think-dataset", + "dataset_revision": "main", + "step": 2362, + "depth": 12, + "target_param_data_ratio": 11.25, + "training_tokens": 1238368256, + "final_sampled_val_bpb": 1.1016183928831433, + "minimum_sampled_val_bpb": 1.1016183928831433, + "full_val_bpb": 1.0504735756248413, + "core_metric": 0.06826829817538614, + "centered_results": { + "hellaswag_zeroshot": 0.03777468204498291, + "jeopardy": 0.0004723665479104966, + "bigbench_qa_wikidata": 0.07568524777889252, + "arc_easy": 0.07968572775522868, + "arc_challenge": -0.06257110834121704, + "copa": 0.019999980926513672, + "commonsense_qa": 0.1134316101670265, + "piqa": 0.0870511531829834, + "openbook_qa": -0.002666652202606201, + "lambada_openai": 0.23073936998844147, + "hellaswag": 0.035384694735209145, + "winograd": 0.07692313194274902, + "winogrande": 0.029202818870544434, + "bigbench_dyck_languages": 0.08300000429153442, + "agi_eval_lsat_ar": 0.08695649355649947, + "bigbench_cs_algorithms": 0.42424240708351135, + "bigbench_operators": 0.07619047909975052, + "bigbench_repeat_copy_logic": 0.0, + "squad": 0.01551560964435339, + "coqa": 0.06776900589466095, + "boolq": -0.15242239676023783, + "bigbench_language_identification": 0.1795379336517636 + }, + "conditioned_samples": [ + { + "prompt": "The capital of France is", + "text": "<|bos|>The capital of France is the capital of the world. \n\nThe capital of the world is the capital of" + }, + { + "prompt": "The chemical symbol of gold is", + "text": "<|bos|>The chemical symbol of gold is a gold chain, and the symbol of the gold chain is a gold chain." + }, + { + "prompt": "If yesterday was Friday, then tomorrow will be", + "text": "<|bos|>If yesterday was Friday, then tomorrow will be Saturday. \n\nI am not sure that I shall be able to go to-morrow" + }, + { + "prompt": "The opposite of hot is", + "text": "<|bos|>The opposite of hot is the opposite of cold. \n\nThe former is the more intense, the latter the" + }, + { + "prompt": "The planets of the solar system are:", + "text": "<|bos|>The planets of the solar system are: \n\n1. The sun, the moon, the stars, the sun, the" + }, + { + "prompt": "My favorite color is", + "text": "<|bos|>My favorite color is a blue, and I have a blue color. \n\nI have a blue color" + }, + { + "prompt": "If 5*x + 3 = 13, then x is", + "text": "<|bos|>If 5*x + 3 = 13, then x is 13, and 13 = 13, and 13 = 13" + } + ], + "unconditioned_samples": [ + "<|bos|>AL PSALTER PSALTER SCRIPTURE PSALTER CHR CHILDREN \n\nChristenome for Modern Sinners Psamates Sallarememorative Psalms Biblioteca \n\nDalmatique Italiano\n\nEaster exegetica Elisha Scotti Netherlandsnio Italiano \n\nDante Bert Raphael Exposition Cant Cant Cant Cant Cant Cant Cant Cant cant\n\nBible Jewish D-Lo spelling Cant Cant \n\nInitiatory Levant Horne\n\nItaliano-Anglo Jupe\n\nFolklore Spanish Olino Florinda In festo \n\nItaliano-Romano Guglielmoi \n\nGui Ser vocali Ministeri Burton", + "<|bos|> salary of the past year, including French and German assistance. The prospects of the business are very gloomy in this country. Rossetti is living, as a manager, in perfect peace. Often, it is said, he sets his empty purse at 73 per cent. It is probable that a very large amount of this is withdrawn every year. But it is probable also that the experience of the last two years is very different. He has been connected with at least one newspaper, having never been in any enterprise except the one that regular news service requires. One incessant fight he made with an old woman who was drugging and disguising herself", + "<|bos|> us Free men and she who could never excuse forgetful Siamese. Well, in the last grand war such men had nothing to lose. Thank you, for your rain, and say 'Good-bye.' I'll all this long. \n\nWith kind Lady Cassoit. \n\nYours very affectionately, \n\nSQUIRE SHANGHANK. \n\nL. SHERgee.\n\n'You were got so devotedly tied up by feeling to this morning that you struggled against it long with all your might when it was taken from me. But, as soon as you had won your way to your old friends and dear home, then I had", + "<|bos|> Kendenthal, Brig.-Gen. \n\nWaller, Mutiny Washington, 9 U.S. 451.\n\nMaryland loss at Manila.-Send prompt orders in case there is any error, the business is completed in twenty days.\n\nAmerican forces at Geba.---Actinon.-S. of the enemy at Geba. C.B. from active duties, natives, tude; valor (Georgia men in hospital), and privates, militia, 3,000 tons War Department. removed shipping lies to the colonies taking contraband of war, removing twelve hundred stone from one stockading", + "<|bos|> HOUSE OF THE LION RIOTS. \n\nAgate.] One who is anxious to get a glimpse into the future of My Majesty in this world, and out of the midst of error, far from denying himself marriage, treats it one who has hitherto trusted himself to carry on the war with difficulty and success: unincorporated, not, as some think, out of discouragement, but because he has shown himself so willing to enter on higher ground; often, not, perhaps, without some little reflection for his own sake, but without question after long and fatiguing deliberation; but, when earnest, unwearied and useful labour rises up", + "<|bos|>1690, 1 August. Governor Nathaniel Claffel. \n\nVan Isle.\n\nU. S. Army. \n\n1 June. General Cooper. \n\n1 June. Colonel Theodore Astley. \n\n1 July. Commander Joseph Palmer.\n\nEntom., Joshua Billings, John Hill, \n\n8 June. Captain John Daniels \n\nPhoebe Talbotton \n\n\"Admiral Porter,\" \"Marguerite\n\nNewburne \n\ntwo or three names for Oct. 14th. \n\nBertie Mills\n\nEmery Park\n\nLaugwanton \n\nCapt. William Ludlow \n\nHezekiah Meyers \n\nMosby", + "<|bos|>Army in the field and defends itself by an effective briinjade. He did not resort to threats, nor to use a poniard; but he only levied dreadfully upon the French who were marching up and down the Army.\n\n\"How should it be practicable now?\" said theLOWERS, while the Army was effecting its march. \"The army should be kept intact. The furnishing of the necessary subsistence should be strictly enforced. Thee should have these river-banks filled up, and garrisons The Executive has orders to inflict extreme corrup-\n\nviction soon on the enemy, because they are", + "<|bos|>the eightieth year of his age.\n\nI had never been conferring, like the paleface VIII. of the Conquests, on the History of \n\nSouth America, and give all the name ever given this island to Penrith.' \n\nLectures to the Greeks. \n\n[See preceding list.] 119-113, and a vaulted great hall.- \n\nApart from the three triumphs above described, there are three books in the Museum that are universally known to the present generation of students. The last has the date \n\n1184.'-Journal of the Institute of Arts, Sciences and Arts, No. 725" + ], + "training_time_seconds": 6288.5381960868835, + "stage_training_flops": 1.0985538644638433e+18, + "inherited_parent_flops": 0.0, + "cumulative_pipeline_training_flops": 1.0985538644638433e+18, + "config_fingerprint": "dd64eb8e857c4d1a", + "git_commit_sha": "48cd5f516123fcf119c7a43a32dd1dfd5c7b8eaa", + "wandb_url": "https://wandb.ai/jbduran-thinkingmachinesncsu/think.nano/runs/46b7f9c8", + "huggingface_url": "https://huggingface.co/jbduran/think.nano/tree/main/experiments/think-d12-r11-run3", + "dataset_fingerprint": "63a5e6be81591d82", + "tokenizer_fingerprint": "85b2a26d1c355860", + "unique_train_tokens": 1275519304, + "effective_epochs": 0.970873786164196 +} diff --git a/experiments/think-d12-r11-run3/tokenizer/experiment_tokenizer.json b/experiments/think-d12-r11-run3/tokenizer/experiment_tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..fc70eca95e86a79db88161e6f68918d1b53cbe91 --- /dev/null +++ b/experiments/think-d12-r11-run3/tokenizer/experiment_tokenizer.json @@ -0,0 +1,18 @@ +{ + "experiment_id": "think-d12-r11-run3", + "dataset": { + "adapter": "parquet_shards", + "repo": "jbduran/think-dataset", + "revision": "main", + "validation_shard": 472, + "num_train_shards": 24, + "download_workers": 4 + }, + "tokenizer": { + "mode": "train", + "max_chars": 2000000000, + "doc_cap": 10000, + "vocab_size": 32768 + }, + "created_at": 1782133420 +} diff --git a/experiments/think-d12-r11-run3/tokenizer/token_bytes.pt b/experiments/think-d12-r11-run3/tokenizer/token_bytes.pt new file mode 100644 index 0000000000000000000000000000000000000000..01d1ec4aab9e8a7d205c3b3ffbeb8da0e9a62db1 --- /dev/null +++ b/experiments/think-d12-r11-run3/tokenizer/token_bytes.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:53db25a1f6969198e67c303e479e97fb49859c715061baec525471666a2f47e1 +size 132649 diff --git a/experiments/think-d12-r11-run3/tokenizer/tokenizer.pkl b/experiments/think-d12-r11-run3/tokenizer/tokenizer.pkl new file mode 100644 index 0000000000000000000000000000000000000000..a17bd392980021628053b95d6425fc556aad527a --- /dev/null +++ b/experiments/think-d12-r11-run3/tokenizer/tokenizer.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:42ea52179dcefb47e4a4bbca7c37f67c3373840a3681587b1c8fe188f00d3bf1 +size 404071 diff --git a/experiments/think-d12-r11.25-ctx4096-sssl/config.json b/experiments/think-d12-r11.25-ctx4096-sssl/config.json new file mode 100644 index 0000000000000000000000000000000000000000..768784a0486010588865a9c7e299d1106131eefc --- /dev/null +++ b/experiments/think-d12-r11.25-ctx4096-sssl/config.json @@ -0,0 +1,58 @@ +{ + "schema_version": 1, + "stage": "base", + "experiment_id": "think-d12-r11.25-ctx4096-sssl", + "dataset": { + "adapter": "parquet_shards", + "repo": "jbduran/think-dataset", + "revision": "main", + "validation_shard": 472, + "num_train_shards": 24, + "download_workers": 4 + }, + "tokenizer": { + "mode": "train", + "max_chars": 2000000000, + "doc_cap": 10000, + "vocab_size": 32768 + }, + "pretokenize": { + "enabled": true, + "slack": 1.03, + "val_tokens": 20971520, + "shard_tokens": 100000000, + "tokenizer_threads": 8 + }, + "training": { + "depth": 12, + "scaling_params": 110100912, + "target_param_data_ratio": 11.25, + "max_seq_len": 4096, + "window_pattern": "SSSL", + "device_batch_size": 8, + "total_batch_size": 524288, + "save_every": 500, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "sample_every": -1 + }, + "artifacts": { + "repo": "jbduran/think.nano" + }, + "wandb": { + "entity": "jbduran-thinkingmachinesncsu", + "project": "think.nano", + "name": "think-d12-r11.25-ctx4096-sssl", + "group": "think-d12", + "tags": [ + "think-dataset", + "d12", + "ratio11.25", + "ctx4096", + "sssl" + ] + }, + "config_fingerprint": "6e9c75a327904b86", + "artifact_path": "experiments/think-d12-r11.25-ctx4096-sssl" +} diff --git a/experiments/think-d12-r11.25-ctx4096-sssl/run.json b/experiments/think-d12-r11.25-ctx4096-sssl/run.json new file mode 100644 index 0000000000000000000000000000000000000000..4ba9f266da85ae23b7ef452be2ffcbd0b4a1b58d --- /dev/null +++ b/experiments/think-d12-r11.25-ctx4096-sssl/run.json @@ -0,0 +1,10 @@ +{ + "experiment_id": "think-d12-r11.25-ctx4096-sssl", + "stage": "base", + "base_experiment_id": "think-d12-r11.25-ctx4096-sssl", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "config_fingerprint": "6e9c75a327904b86", + "wandb_run_id": "d925b571", + "created_at": 1784302194 +} diff --git a/experiments/think-d12-r11.25-ctx4096-sssl/tokenizer/experiment_tokenizer.json b/experiments/think-d12-r11.25-ctx4096-sssl/tokenizer/experiment_tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..b492eed0c58653d83091c8723e1267fde869e135 --- /dev/null +++ b/experiments/think-d12-r11.25-ctx4096-sssl/tokenizer/experiment_tokenizer.json @@ -0,0 +1,18 @@ +{ + "experiment_id": "think-d12-r11.25-ctx4096-sssl", + "dataset": { + "adapter": "parquet_shards", + "repo": "jbduran/think-dataset", + "revision": "main", + "validation_shard": 472, + "num_train_shards": 24, + "download_workers": 4 + }, + "tokenizer": { + "mode": "train", + "max_chars": 2000000000, + "doc_cap": 10000, + "vocab_size": 32768 + }, + "created_at": 1784302214 +} diff --git a/experiments/think-d12-r11.25-ctx4096-sssl/tokenizer/token_bytes.pt b/experiments/think-d12-r11.25-ctx4096-sssl/tokenizer/token_bytes.pt new file mode 100644 index 0000000000000000000000000000000000000000..52b8c6160971208bdb8a09f45e79882217012ba5 --- /dev/null +++ b/experiments/think-d12-r11.25-ctx4096-sssl/tokenizer/token_bytes.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:89a69da6264286005b0e65be12a1afc9bc44936d535527f347d22ab74790076c +size 132649 diff --git a/experiments/think-d12-r11.25-ctx4096-sssl/tokenizer/tokenizer.pkl b/experiments/think-d12-r11.25-ctx4096-sssl/tokenizer/tokenizer.pkl new file mode 100644 index 0000000000000000000000000000000000000000..3d323d48ca5e52338e432ef2a6ed24f12b15c4e8 --- /dev/null +++ b/experiments/think-d12-r11.25-ctx4096-sssl/tokenizer/tokenizer.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2df5bd08e7e921a0404abe4642907707baa66c6840e74a90aa16d16b268bd5d1 +size 404037 diff --git a/experiments/think-d12-r11.25-ctx4096/base_checkpoints/meta_000500.json b/experiments/think-d12-r11.25-ctx4096/base_checkpoints/meta_000500.json new file mode 100644 index 0000000000000000000000000000000000000000..621a745b56f619d244c4cebe618fab1a46780a38 --- /dev/null +++ b/experiments/think-d12-r11.25-ctx4096/base_checkpoints/meta_000500.json @@ -0,0 +1,140 @@ +{ + "step": 500, + "training_complete": false, + "experiment_id": "think-d12-r11.25-ctx4096", + "val_bpb": 1.3085028476600495, + "model_config": { + "sequence_len": 4096, + "vocab_size": 32768, + "n_layer": 12, + "n_head": 6, + "n_kv_head": 6, + "n_embd": 768, + "window_pattern": "L" + }, + "user_config": { + "run": "think-d12-r11.25-ctx4096", + "wandb_run_id": "85989fb4", + "wandb_group": "think-d12", + "wandb_tags": "think-dataset,d12,ratio11.25,ctx4096", + "device_type": "", + "fp8": false, + "fp8_recipe": "tensorwise", + "depth": 12, + "aspect_ratio": 64, + "head_dim": 128, + "max_seq_len": 4096, + "window_pattern": "L", + "num_iterations": -1, + "target_flops": -1.0, + "target_param_data_ratio": 11.25, + "device_batch_size": 8, + "total_batch_size": 524288, + "embedding_lr": 0.3, + "unembedding_lr": 0.008, + "weight_decay": 0.28, + "matrix_lr": 0.02, + "scalar_lr": 0.5, + "warmup_steps": 40, + "warmdown_ratio": 0.65, + "final_lr_frac": 0.05, + "resume_from_step": -1, + "pretokenized": true, + "data_dir": "/content/nanochat_cache/experiments/think-d12-r11.25-ctx4096/data", + "tokenizer_dir": "/content/nanochat_cache/experiments/think-d12-r11.25-ctx4096/tokenizer", + "pretokenized_dir": "/content/nanochat_cache/experiments/think-d12-r11.25-ctx4096/pretok", + "checkpoint_dir": "/content/nanochat_cache/experiments/think-d12-r11.25-ctx4096/base_checkpoints", + "experiment_id": "think-d12-r11.25-ctx4096", + "experiment_config": "/content/nanochat_cache/experiments/think-d12-r11.25-ctx4096/config.json", + "tokenizer_fingerprint": "7ad989ceea17794b", + "git_commit_sha": "128fb5d3c7f5e7122b6eb4390cf207e7e986d9dd", + "seed": 42, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "core_metric_max_per_task": 500, + "sample_every": -1, + "save_every": 500, + "model_tag": "think-d12-r11.25-ctx4096", + "experiment": { + "schema_version": 1, + "stage": "base", + "experiment_id": "think-d12-r11.25-ctx4096", + "dataset": { + "adapter": "parquet_shards", + "repo": "jbduran/think-dataset", + "revision": "main", + "validation_shard": 472, + "num_train_shards": 24, + "download_workers": 4 + }, + "tokenizer": { + "mode": "train", + "max_chars": 2000000000, + "doc_cap": 10000, + "vocab_size": 32768 + }, + "pretokenize": { + "enabled": true, + "slack": 1.03, + "val_tokens": 20971520, + "shard_tokens": 100000000, + "tokenizer_threads": 8 + }, + "training": { + "depth": 12, + "scaling_params": 110100912, + "target_param_data_ratio": 11.25, + "max_seq_len": 4096, + "window_pattern": "L", + "device_batch_size": 8, + "total_batch_size": 524288, + "save_every": 500, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "sample_every": -1 + }, + "artifacts": { + "repo": "jbduran/think.nano" + }, + "wandb": { + "entity": "jbduran-thinkingmachinesncsu", + "project": "think.nano", + "name": "think-d12-r11.25-ctx4096", + "group": "think-d12", + "tags": [ + "think-dataset", + "d12", + "ratio11.25", + "ctx4096" + ] + }, + "config_fingerprint": "a1e3ff2a651b87ea", + "artifact_path": "experiments/think-d12-r11.25-ctx4096" + }, + "stage": "base", + "base_experiment_id": "think-d12-r11.25-ctx4096", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "config_fingerprint": "a1e3ff2a651b87ea" + }, + "device_batch_size": 8, + "max_seq_len": 4096, + "total_batch_size": 524288, + "dataloader_state_dict": { + "file_idx": 2, + "pos": 62184769, + "epoch": 1, + "pq_idx": 2, + "rg_idx": 62184769 + }, + "loop_state": { + "min_val_bpb": 1.3085028476600495, + "smooth_train_loss": 3.5543985215236638, + "total_training_time": 1486.219638824463, + "stage_training_flops": 291921016651776000, + "inherited_parent_flops": 0.0, + "cumulative_pipeline_training_flops": 291921016651776000 + } +} \ No newline at end of file diff --git a/experiments/think-d12-r11.25-ctx4096/base_checkpoints/meta_001000.json b/experiments/think-d12-r11.25-ctx4096/base_checkpoints/meta_001000.json new file mode 100644 index 0000000000000000000000000000000000000000..8ff79e207d7bf5cf702bdb5fd2f9761ea2a360fb --- /dev/null +++ b/experiments/think-d12-r11.25-ctx4096/base_checkpoints/meta_001000.json @@ -0,0 +1,140 @@ +{ + "step": 1000, + "training_complete": false, + "experiment_id": "think-d12-r11.25-ctx4096", + "val_bpb": 1.209916555955143, + "model_config": { + "sequence_len": 4096, + "vocab_size": 32768, + "n_layer": 12, + "n_head": 6, + "n_kv_head": 6, + "n_embd": 768, + "window_pattern": "L" + }, + "user_config": { + "run": "think-d12-r11.25-ctx4096", + "wandb_run_id": "85989fb4", + "wandb_group": "think-d12", + "wandb_tags": "think-dataset,d12,ratio11.25,ctx4096", + "device_type": "", + "fp8": false, + "fp8_recipe": "tensorwise", + "depth": 12, + "aspect_ratio": 64, + "head_dim": 128, + "max_seq_len": 4096, + "window_pattern": "L", + "num_iterations": -1, + "target_flops": -1.0, + "target_param_data_ratio": 11.25, + "device_batch_size": 8, + "total_batch_size": 524288, + "embedding_lr": 0.3, + "unembedding_lr": 0.008, + "weight_decay": 0.28, + "matrix_lr": 0.02, + "scalar_lr": 0.5, + "warmup_steps": 40, + "warmdown_ratio": 0.65, + "final_lr_frac": 0.05, + "resume_from_step": 500, + "pretokenized": true, + "data_dir": "/content/nanochat_cache/experiments/think-d12-r11.25-ctx4096/data", + "tokenizer_dir": "/content/nanochat_cache/experiments/think-d12-r11.25-ctx4096/tokenizer", + "pretokenized_dir": "/content/nanochat_cache/experiments/think-d12-r11.25-ctx4096/pretok", + "checkpoint_dir": "/content/nanochat_cache/experiments/think-d12-r11.25-ctx4096/base_checkpoints", + "experiment_id": "think-d12-r11.25-ctx4096", + "experiment_config": "/content/nanochat_cache/experiments/think-d12-r11.25-ctx4096/config.json", + "tokenizer_fingerprint": "7ad989ceea17794b", + "git_commit_sha": "128fb5d3c7f5e7122b6eb4390cf207e7e986d9dd", + "seed": 42, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "core_metric_max_per_task": 500, + "sample_every": -1, + "save_every": 500, + "model_tag": "think-d12-r11.25-ctx4096", + "experiment": { + "schema_version": 1, + "stage": "base", + "experiment_id": "think-d12-r11.25-ctx4096", + "dataset": { + "adapter": "parquet_shards", + "repo": "jbduran/think-dataset", + "revision": "main", + "validation_shard": 472, + "num_train_shards": 24, + "download_workers": 4 + }, + "tokenizer": { + "mode": "train", + "max_chars": 2000000000, + "doc_cap": 10000, + "vocab_size": 32768 + }, + "pretokenize": { + "enabled": true, + "slack": 1.03, + "val_tokens": 20971520, + "shard_tokens": 100000000, + "tokenizer_threads": 8 + }, + "training": { + "depth": 12, + "scaling_params": 110100912, + "target_param_data_ratio": 11.25, + "max_seq_len": 4096, + "window_pattern": "L", + "device_batch_size": 8, + "total_batch_size": 524288, + "save_every": 500, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "sample_every": -1 + }, + "artifacts": { + "repo": "jbduran/think.nano" + }, + "wandb": { + "entity": "jbduran-thinkingmachinesncsu", + "project": "think.nano", + "name": "think-d12-r11.25-ctx4096", + "group": "think-d12", + "tags": [ + "think-dataset", + "d12", + "ratio11.25", + "ctx4096" + ] + }, + "config_fingerprint": "a1e3ff2a651b87ea", + "artifact_path": "experiments/think-d12-r11.25-ctx4096" + }, + "stage": "base", + "base_experiment_id": "think-d12-r11.25-ctx4096", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "config_fingerprint": "a1e3ff2a651b87ea" + }, + "device_batch_size": 8, + "max_seq_len": 4096, + "total_batch_size": 524288, + "dataloader_state_dict": { + "file_idx": 5, + "pos": 24369538, + "epoch": 1, + "pq_idx": 5, + "rg_idx": 24369538 + }, + "loop_state": { + "min_val_bpb": 1.209916555955143, + "smooth_train_loss": 3.4094984750904938, + "total_training_time": 3055.330541372299, + "stage_training_flops": 583842033303552000, + "inherited_parent_flops": 0.0, + "cumulative_pipeline_training_flops": 583842033303552000 + } +} \ No newline at end of file diff --git a/experiments/think-d12-r11.25-ctx4096/base_checkpoints/meta_001500.json b/experiments/think-d12-r11.25-ctx4096/base_checkpoints/meta_001500.json new file mode 100644 index 0000000000000000000000000000000000000000..ddbfd68db055843968eb36669e5c791ed83bd0e4 --- /dev/null +++ b/experiments/think-d12-r11.25-ctx4096/base_checkpoints/meta_001500.json @@ -0,0 +1,140 @@ +{ + "step": 1500, + "training_complete": false, + "experiment_id": "think-d12-r11.25-ctx4096", + "val_bpb": 1.1509853231334473, + "model_config": { + "sequence_len": 4096, + "vocab_size": 32768, + "n_layer": 12, + "n_head": 6, + "n_kv_head": 6, + "n_embd": 768, + "window_pattern": "L" + }, + "user_config": { + "run": "think-d12-r11.25-ctx4096", + "wandb_run_id": "85989fb4", + "wandb_group": "think-d12", + "wandb_tags": "think-dataset,d12,ratio11.25,ctx4096", + "device_type": "", + "fp8": false, + "fp8_recipe": "tensorwise", + "depth": 12, + "aspect_ratio": 64, + "head_dim": 128, + "max_seq_len": 4096, + "window_pattern": "L", + "num_iterations": -1, + "target_flops": -1.0, + "target_param_data_ratio": 11.25, + "device_batch_size": 8, + "total_batch_size": 524288, + "embedding_lr": 0.3, + "unembedding_lr": 0.008, + "weight_decay": 0.28, + "matrix_lr": 0.02, + "scalar_lr": 0.5, + "warmup_steps": 40, + "warmdown_ratio": 0.65, + "final_lr_frac": 0.05, + "resume_from_step": 500, + "pretokenized": true, + "data_dir": "/content/nanochat_cache/experiments/think-d12-r11.25-ctx4096/data", + "tokenizer_dir": "/content/nanochat_cache/experiments/think-d12-r11.25-ctx4096/tokenizer", + "pretokenized_dir": "/content/nanochat_cache/experiments/think-d12-r11.25-ctx4096/pretok", + "checkpoint_dir": "/content/nanochat_cache/experiments/think-d12-r11.25-ctx4096/base_checkpoints", + "experiment_id": "think-d12-r11.25-ctx4096", + "experiment_config": "/content/nanochat_cache/experiments/think-d12-r11.25-ctx4096/config.json", + "tokenizer_fingerprint": "7ad989ceea17794b", + "git_commit_sha": "128fb5d3c7f5e7122b6eb4390cf207e7e986d9dd", + "seed": 42, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "core_metric_max_per_task": 500, + "sample_every": -1, + "save_every": 500, + "model_tag": "think-d12-r11.25-ctx4096", + "experiment": { + "schema_version": 1, + "stage": "base", + "experiment_id": "think-d12-r11.25-ctx4096", + "dataset": { + "adapter": "parquet_shards", + "repo": "jbduran/think-dataset", + "revision": "main", + "validation_shard": 472, + "num_train_shards": 24, + "download_workers": 4 + }, + "tokenizer": { + "mode": "train", + "max_chars": 2000000000, + "doc_cap": 10000, + "vocab_size": 32768 + }, + "pretokenize": { + "enabled": true, + "slack": 1.03, + "val_tokens": 20971520, + "shard_tokens": 100000000, + "tokenizer_threads": 8 + }, + "training": { + "depth": 12, + "scaling_params": 110100912, + "target_param_data_ratio": 11.25, + "max_seq_len": 4096, + "window_pattern": "L", + "device_batch_size": 8, + "total_batch_size": 524288, + "save_every": 500, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "sample_every": -1 + }, + "artifacts": { + "repo": "jbduran/think.nano" + }, + "wandb": { + "entity": "jbduran-thinkingmachinesncsu", + "project": "think.nano", + "name": "think-d12-r11.25-ctx4096", + "group": "think-d12", + "tags": [ + "think-dataset", + "d12", + "ratio11.25", + "ctx4096" + ] + }, + "config_fingerprint": "a1e3ff2a651b87ea", + "artifact_path": "experiments/think-d12-r11.25-ctx4096" + }, + "stage": "base", + "base_experiment_id": "think-d12-r11.25-ctx4096", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "config_fingerprint": "a1e3ff2a651b87ea" + }, + "device_batch_size": 8, + "max_seq_len": 4096, + "total_batch_size": 524288, + "dataloader_state_dict": { + "file_idx": 7, + "pos": 86521538, + "epoch": 1, + "pq_idx": 7, + "rg_idx": 86521538 + }, + "loop_state": { + "min_val_bpb": 1.1509853231334473, + "smooth_train_loss": 3.2151429122859003, + "total_training_time": 4564.625297546387, + "stage_training_flops": 875763049955328000, + "inherited_parent_flops": 0.0, + "cumulative_pipeline_training_flops": 875763049955328000 + } +} \ No newline at end of file diff --git a/experiments/think-d12-r11.25-ctx4096/base_checkpoints/meta_002000.json b/experiments/think-d12-r11.25-ctx4096/base_checkpoints/meta_002000.json new file mode 100644 index 0000000000000000000000000000000000000000..102758e5e738a387460507ad0661b7ef5c899772 --- /dev/null +++ b/experiments/think-d12-r11.25-ctx4096/base_checkpoints/meta_002000.json @@ -0,0 +1,140 @@ +{ + "step": 2000, + "training_complete": false, + "experiment_id": "think-d12-r11.25-ctx4096", + "val_bpb": 1.099780461697027, + "model_config": { + "sequence_len": 4096, + "vocab_size": 32768, + "n_layer": 12, + "n_head": 6, + "n_kv_head": 6, + "n_embd": 768, + "window_pattern": "L" + }, + "user_config": { + "run": "think-d12-r11.25-ctx4096", + "wandb_run_id": "85989fb4", + "wandb_group": "think-d12", + "wandb_tags": "think-dataset,d12,ratio11.25,ctx4096", + "device_type": "", + "fp8": false, + "fp8_recipe": "tensorwise", + "depth": 12, + "aspect_ratio": 64, + "head_dim": 128, + "max_seq_len": 4096, + "window_pattern": "L", + "num_iterations": -1, + "target_flops": -1.0, + "target_param_data_ratio": 11.25, + "device_batch_size": 8, + "total_batch_size": 524288, + "embedding_lr": 0.3, + "unembedding_lr": 0.008, + "weight_decay": 0.28, + "matrix_lr": 0.02, + "scalar_lr": 0.5, + "warmup_steps": 40, + "warmdown_ratio": 0.65, + "final_lr_frac": 0.05, + "resume_from_step": 500, + "pretokenized": true, + "data_dir": "/content/nanochat_cache/experiments/think-d12-r11.25-ctx4096/data", + "tokenizer_dir": "/content/nanochat_cache/experiments/think-d12-r11.25-ctx4096/tokenizer", + "pretokenized_dir": "/content/nanochat_cache/experiments/think-d12-r11.25-ctx4096/pretok", + "checkpoint_dir": "/content/nanochat_cache/experiments/think-d12-r11.25-ctx4096/base_checkpoints", + "experiment_id": "think-d12-r11.25-ctx4096", + "experiment_config": "/content/nanochat_cache/experiments/think-d12-r11.25-ctx4096/config.json", + "tokenizer_fingerprint": "7ad989ceea17794b", + "git_commit_sha": "128fb5d3c7f5e7122b6eb4390cf207e7e986d9dd", + "seed": 42, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "core_metric_max_per_task": 500, + "sample_every": -1, + "save_every": 500, + "model_tag": "think-d12-r11.25-ctx4096", + "experiment": { + "schema_version": 1, + "stage": "base", + "experiment_id": "think-d12-r11.25-ctx4096", + "dataset": { + "adapter": "parquet_shards", + "repo": "jbduran/think-dataset", + "revision": "main", + "validation_shard": 472, + "num_train_shards": 24, + "download_workers": 4 + }, + "tokenizer": { + "mode": "train", + "max_chars": 2000000000, + "doc_cap": 10000, + "vocab_size": 32768 + }, + "pretokenize": { + "enabled": true, + "slack": 1.03, + "val_tokens": 20971520, + "shard_tokens": 100000000, + "tokenizer_threads": 8 + }, + "training": { + "depth": 12, + "scaling_params": 110100912, + "target_param_data_ratio": 11.25, + "max_seq_len": 4096, + "window_pattern": "L", + "device_batch_size": 8, + "total_batch_size": 524288, + "save_every": 500, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "sample_every": -1 + }, + "artifacts": { + "repo": "jbduran/think.nano" + }, + "wandb": { + "entity": "jbduran-thinkingmachinesncsu", + "project": "think.nano", + "name": "think-d12-r11.25-ctx4096", + "group": "think-d12", + "tags": [ + "think-dataset", + "d12", + "ratio11.25", + "ctx4096" + ] + }, + "config_fingerprint": "a1e3ff2a651b87ea", + "artifact_path": "experiments/think-d12-r11.25-ctx4096" + }, + "stage": "base", + "base_experiment_id": "think-d12-r11.25-ctx4096", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "config_fingerprint": "a1e3ff2a651b87ea" + }, + "device_batch_size": 8, + "max_seq_len": 4096, + "total_batch_size": 524288, + "dataloader_state_dict": { + "file_idx": 10, + "pos": 48673538, + "epoch": 1, + "pq_idx": 10, + "rg_idx": 48673538 + }, + "loop_state": { + "min_val_bpb": 1.099780461697027, + "smooth_train_loss": 3.1695909607341712, + "total_training_time": 6075.22934627533, + "stage_training_flops": 1167684066607104000, + "inherited_parent_flops": 0.0, + "cumulative_pipeline_training_flops": 1167684066607104000 + } +} \ No newline at end of file diff --git a/experiments/think-d12-r11.25-ctx4096/base_checkpoints/meta_002362.json b/experiments/think-d12-r11.25-ctx4096/base_checkpoints/meta_002362.json new file mode 100644 index 0000000000000000000000000000000000000000..65a6a0092c2c3e6b33a4c88fe36c0606a1454473 --- /dev/null +++ b/experiments/think-d12-r11.25-ctx4096/base_checkpoints/meta_002362.json @@ -0,0 +1,140 @@ +{ + "step": 2362, + "training_complete": true, + "experiment_id": "think-d12-r11.25-ctx4096", + "val_bpb": 1.0795015991364572, + "model_config": { + "sequence_len": 4096, + "vocab_size": 32768, + "n_layer": 12, + "n_head": 6, + "n_kv_head": 6, + "n_embd": 768, + "window_pattern": "L" + }, + "user_config": { + "run": "think-d12-r11.25-ctx4096", + "wandb_run_id": "85989fb4", + "wandb_group": "think-d12", + "wandb_tags": "think-dataset,d12,ratio11.25,ctx4096", + "device_type": "", + "fp8": false, + "fp8_recipe": "tensorwise", + "depth": 12, + "aspect_ratio": 64, + "head_dim": 128, + "max_seq_len": 4096, + "window_pattern": "L", + "num_iterations": -1, + "target_flops": -1.0, + "target_param_data_ratio": 11.25, + "device_batch_size": 8, + "total_batch_size": 524288, + "embedding_lr": 0.3, + "unembedding_lr": 0.008, + "weight_decay": 0.28, + "matrix_lr": 0.02, + "scalar_lr": 0.5, + "warmup_steps": 40, + "warmdown_ratio": 0.65, + "final_lr_frac": 0.05, + "resume_from_step": 500, + "pretokenized": true, + "data_dir": "/content/nanochat_cache/experiments/think-d12-r11.25-ctx4096/data", + "tokenizer_dir": "/content/nanochat_cache/experiments/think-d12-r11.25-ctx4096/tokenizer", + "pretokenized_dir": "/content/nanochat_cache/experiments/think-d12-r11.25-ctx4096/pretok", + "checkpoint_dir": "/content/nanochat_cache/experiments/think-d12-r11.25-ctx4096/base_checkpoints", + "experiment_id": "think-d12-r11.25-ctx4096", + "experiment_config": "/content/nanochat_cache/experiments/think-d12-r11.25-ctx4096/config.json", + "tokenizer_fingerprint": "7ad989ceea17794b", + "git_commit_sha": "128fb5d3c7f5e7122b6eb4390cf207e7e986d9dd", + "seed": 42, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "core_metric_max_per_task": 500, + "sample_every": -1, + "save_every": 500, + "model_tag": "think-d12-r11.25-ctx4096", + "experiment": { + "schema_version": 1, + "stage": "base", + "experiment_id": "think-d12-r11.25-ctx4096", + "dataset": { + "adapter": "parquet_shards", + "repo": "jbduran/think-dataset", + "revision": "main", + "validation_shard": 472, + "num_train_shards": 24, + "download_workers": 4 + }, + "tokenizer": { + "mode": "train", + "max_chars": 2000000000, + "doc_cap": 10000, + "vocab_size": 32768 + }, + "pretokenize": { + "enabled": true, + "slack": 1.03, + "val_tokens": 20971520, + "shard_tokens": 100000000, + "tokenizer_threads": 8 + }, + "training": { + "depth": 12, + "scaling_params": 110100912, + "target_param_data_ratio": 11.25, + "max_seq_len": 4096, + "window_pattern": "L", + "device_batch_size": 8, + "total_batch_size": 524288, + "save_every": 500, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "sample_every": -1 + }, + "artifacts": { + "repo": "jbduran/think.nano" + }, + "wandb": { + "entity": "jbduran-thinkingmachinesncsu", + "project": "think.nano", + "name": "think-d12-r11.25-ctx4096", + "group": "think-d12", + "tags": [ + "think-dataset", + "d12", + "ratio11.25", + "ctx4096" + ] + }, + "config_fingerprint": "a1e3ff2a651b87ea", + "artifact_path": "experiments/think-d12-r11.25-ctx4096" + }, + "stage": "base", + "base_experiment_id": "think-d12-r11.25-ctx4096", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "config_fingerprint": "a1e3ff2a651b87ea" + }, + "device_batch_size": 8, + "max_seq_len": 4096, + "total_batch_size": 524288, + "dataloader_state_dict": { + "file_idx": 12, + "pos": 38471586, + "epoch": 1, + "pq_idx": 12, + "rg_idx": 38471586 + }, + "loop_state": { + "min_val_bpb": 1.0795015991364572, + "smooth_train_loss": 3.0619805740794512, + "total_training_time": 7167.931929588318, + "stage_training_flops": 1379034882662989824, + "inherited_parent_flops": 0.0, + "cumulative_pipeline_training_flops": 1379034882662989824 + } +} \ No newline at end of file diff --git a/experiments/think-d12-r11.25-ctx4096/base_checkpoints/model_000500.pt b/experiments/think-d12-r11.25-ctx4096/base_checkpoints/model_000500.pt new file mode 100644 index 0000000000000000000000000000000000000000..4c3181f54124f5ba051f2c3b4dc0f973adb5ceb2 --- /dev/null +++ b/experiments/think-d12-r11.25-ctx4096/base_checkpoints/model_000500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eae964a0bb2c1375c3b4451c8aa5570663349d91ee3b321c472df0d3a4660219 +size 792761690 diff --git a/experiments/think-d12-r11.25-ctx4096/base_checkpoints/model_001000.pt b/experiments/think-d12-r11.25-ctx4096/base_checkpoints/model_001000.pt new file mode 100644 index 0000000000000000000000000000000000000000..571aa5a944ca0b8b5889c94a67706257db85131e --- /dev/null +++ b/experiments/think-d12-r11.25-ctx4096/base_checkpoints/model_001000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a66e11e92bed0abf8702c6553dfaed10a232a99c9b6e9d90303d5847430adc1d +size 792761690 diff --git a/experiments/think-d12-r11.25-ctx4096/base_checkpoints/model_001500.pt b/experiments/think-d12-r11.25-ctx4096/base_checkpoints/model_001500.pt new file mode 100644 index 0000000000000000000000000000000000000000..cbd23725cea6e13eb84db458267ace1699d65d01 --- /dev/null +++ b/experiments/think-d12-r11.25-ctx4096/base_checkpoints/model_001500.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:07b77aef3943e2b0cb0434580c50489846faaefcf579df38c79139f1f1b16093 +size 792761690 diff --git a/experiments/think-d12-r11.25-ctx4096/base_checkpoints/model_002000.pt b/experiments/think-d12-r11.25-ctx4096/base_checkpoints/model_002000.pt new file mode 100644 index 0000000000000000000000000000000000000000..7e23fb9eaae0e92d29d0472571dd2fff20107b47 --- /dev/null +++ b/experiments/think-d12-r11.25-ctx4096/base_checkpoints/model_002000.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c26eb6938cd1b71b2f3b66d7287f6bc768184242413062ece92523a88694d434 +size 792761690 diff --git a/experiments/think-d12-r11.25-ctx4096/base_checkpoints/model_002362.pt b/experiments/think-d12-r11.25-ctx4096/base_checkpoints/model_002362.pt new file mode 100644 index 0000000000000000000000000000000000000000..07ea2f582c28be0bebb4d785113f53a6af83b1dc --- /dev/null +++ b/experiments/think-d12-r11.25-ctx4096/base_checkpoints/model_002362.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5925ab0fb833c557d0252f792cc84f5b538514defc131f2dae5eb3ebcd038705 +size 792761690 diff --git a/experiments/think-d12-r11.25-ctx4096/base_checkpoints/optim_000500_rank0.pt b/experiments/think-d12-r11.25-ctx4096/base_checkpoints/optim_000500_rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..f1e7adecb5041fd09f6e1a50f21df76a741ccc9c --- /dev/null +++ b/experiments/think-d12-r11.25-ctx4096/base_checkpoints/optim_000500_rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:df415f6eb227a7508933cf3e56d35b580e1c4fc294b213ab3913340595f5cfdc +size 1246165357 diff --git a/experiments/think-d12-r11.25-ctx4096/base_checkpoints/optim_001000_rank0.pt b/experiments/think-d12-r11.25-ctx4096/base_checkpoints/optim_001000_rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..2a421f2700606a3cedb0c4b930ea2b10ef13ed3a --- /dev/null +++ b/experiments/think-d12-r11.25-ctx4096/base_checkpoints/optim_001000_rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3039b07883f2b42253768f446c4c007427398b7b019300365ea7be9e0fc42ad5 +size 1246165357 diff --git a/experiments/think-d12-r11.25-ctx4096/base_checkpoints/optim_001500_rank0.pt b/experiments/think-d12-r11.25-ctx4096/base_checkpoints/optim_001500_rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..793b9df065ed3fe1868036306fcd30952919c8f4 --- /dev/null +++ b/experiments/think-d12-r11.25-ctx4096/base_checkpoints/optim_001500_rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2407aa8a07d26bd2f67674ecd6849bea4ba0da3577680f8b1fdba446ac0e1fe8 +size 1246165357 diff --git a/experiments/think-d12-r11.25-ctx4096/base_checkpoints/optim_002000_rank0.pt b/experiments/think-d12-r11.25-ctx4096/base_checkpoints/optim_002000_rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..5ad8110c17ac5e0fde78cad1a9f90d34136edec6 --- /dev/null +++ b/experiments/think-d12-r11.25-ctx4096/base_checkpoints/optim_002000_rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2d22fa6ba43269e9464c107e377955f93e7032ca892a20961f6dae3ab8800b2f +size 1246165357 diff --git a/experiments/think-d12-r11.25-ctx4096/base_checkpoints/optim_002362_rank0.pt b/experiments/think-d12-r11.25-ctx4096/base_checkpoints/optim_002362_rank0.pt new file mode 100644 index 0000000000000000000000000000000000000000..1bafee36bb702704c1b5ab8e22812dcb4369adbc --- /dev/null +++ b/experiments/think-d12-r11.25-ctx4096/base_checkpoints/optim_002362_rank0.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:59ebb8f16f64fdc23ef0136ea29fd602db7eb76ed53c0b4b36167d1b65ae06dc +size 1246165357 diff --git a/experiments/think-d12-r11.25-ctx4096/config.json b/experiments/think-d12-r11.25-ctx4096/config.json new file mode 100644 index 0000000000000000000000000000000000000000..c0b1edb621d47cf9c5eb14744f0c1d0db7ffa254 --- /dev/null +++ b/experiments/think-d12-r11.25-ctx4096/config.json @@ -0,0 +1,57 @@ +{ + "schema_version": 1, + "stage": "base", + "experiment_id": "think-d12-r11.25-ctx4096", + "dataset": { + "adapter": "parquet_shards", + "repo": "jbduran/think-dataset", + "revision": "main", + "validation_shard": 472, + "num_train_shards": 24, + "download_workers": 4 + }, + "tokenizer": { + "mode": "train", + "max_chars": 2000000000, + "doc_cap": 10000, + "vocab_size": 32768 + }, + "pretokenize": { + "enabled": true, + "slack": 1.03, + "val_tokens": 20971520, + "shard_tokens": 100000000, + "tokenizer_threads": 8 + }, + "training": { + "depth": 12, + "scaling_params": 110100912, + "target_param_data_ratio": 11.25, + "max_seq_len": 4096, + "window_pattern": "L", + "device_batch_size": 8, + "total_batch_size": 524288, + "save_every": 500, + "eval_every": 250, + "eval_tokens": 2097152, + "core_metric_every": -1, + "sample_every": -1 + }, + "artifacts": { + "repo": "jbduran/think.nano" + }, + "wandb": { + "entity": "jbduran-thinkingmachinesncsu", + "project": "think.nano", + "name": "think-d12-r11.25-ctx4096", + "group": "think-d12", + "tags": [ + "think-dataset", + "d12", + "ratio11.25", + "ctx4096" + ] + }, + "config_fingerprint": "a1e3ff2a651b87ea", + "artifact_path": "experiments/think-d12-r11.25-ctx4096" +} diff --git a/experiments/think-d12-r11.25-ctx4096/evals/core.json b/experiments/think-d12-r11.25-ctx4096/evals/core.json new file mode 100644 index 0000000000000000000000000000000000000000..18bd17537653676d6499795eafc1facad7c7104d --- /dev/null +++ b/experiments/think-d12-r11.25-ctx4096/evals/core.json @@ -0,0 +1,56 @@ +{ + "model": "base_model (step 2362)", + "step": 2362, + "bpb": {}, + "core_metric": 0.06440192842734686, + "core_results": { + "hellaswag_zeroshot": 0.2777335047721863, + "jeopardy": 0.0009447330958209932, + "bigbench_qa_wikidata": 0.09128487855195999, + "arc_easy": 0.31186866760253906, + "arc_challenge": 0.21160408854484558, + "copa": 0.5099999904632568, + "commonsense_qa": 0.31449630856513977, + "piqa": 0.5369967222213745, + "openbook_qa": 0.24800001084804535, + "lambada_openai": 0.23015718162059784, + "hellaswag": 0.2796255648136139, + "winograd": 0.553113579750061, + "winogrande": 0.5106551051139832, + "bigbench_dyck_languages": 0.10200000554323196, + "agi_eval_lsat_ar": 0.2956521511077881, + "bigbench_cs_algorithms": 0.40303027629852295, + "bigbench_operators": 0.0714285746216774, + "bigbench_repeat_copy_logic": 0.0, + "squad": 0.02308420091867447, + "coqa": 0.06664161384105682, + "boolq": 0.49082568287849426, + "bigbench_language_identification": 0.25360000133514404 + }, + "centered_results": { + "hellaswag_zeroshot": 0.03697800636291504, + "jeopardy": 0.0009447330958209932, + "bigbench_qa_wikidata": 0.09128487855195999, + "arc_easy": 0.08249155680338542, + "arc_challenge": -0.05119454860687256, + "copa": 0.019999980926513672, + "commonsense_qa": 0.14312038570642469, + "piqa": 0.07399344444274902, + "openbook_qa": -0.002666652202606201, + "lambada_openai": 0.23015718162059784, + "hellaswag": 0.03950075308481852, + "winograd": 0.10622715950012207, + "winogrande": 0.02131021022796631, + "bigbench_dyck_languages": 0.10200000554323196, + "agi_eval_lsat_ar": 0.1195651888847351, + "bigbench_cs_algorithms": 0.40303027629852295, + "bigbench_operators": 0.0714285746216774, + "bigbench_repeat_copy_logic": 0.0, + "squad": 0.02308420091867447, + "coqa": 0.06664161384105682, + "boolq": -0.3399324134776467, + "bigbench_language_identification": 0.17887788925758422 + }, + "conditioned_samples": [], + "unconditioned_samples": [] +} \ No newline at end of file diff --git a/experiments/think-d12-r11.25-ctx4096/evals/samples.json b/experiments/think-d12-r11.25-ctx4096/evals/samples.json new file mode 100644 index 0000000000000000000000000000000000000000..bc023780539cb6a95be1c927a096d903f9c1dc3c --- /dev/null +++ b/experiments/think-d12-r11.25-ctx4096/evals/samples.json @@ -0,0 +1,48 @@ +{ + "model": "base_model (step 2362)", + "step": 2362, + "bpb": {}, + "core_metric": null, + "core_results": null, + "centered_results": null, + "conditioned_samples": [ + { + "prompt": "The capital of France is", + "text": "<|bos|>The capital of France is not yet exhausted, and the French are not yet in the field. \n\nThe" + }, + { + "prompt": "The chemical symbol of gold is", + "text": "<|bos|>The chemical symbol of gold is the symbol of the gold of the earth. \n\nThe gold of the earth is" + }, + { + "prompt": "If yesterday was Friday, then tomorrow will be", + "text": "<|bos|>If yesterday was Friday, then tomorrow will be the day of the Lord's coming. \n\nThe Lord will be with you," + }, + { + "prompt": "The opposite of hot is", + "text": "<|bos|>The opposite of hot is the opposite of cold. \n\nThe former is the opposite of cold, and the" + }, + { + "prompt": "The planets of the solar system are:", + "text": "<|bos|>The planets of the solar system are: \n\n1. The sun, which is the centre of the solar system, and" + }, + { + "prompt": "My favorite color is", + "text": "<|bos|>My favorite color is the color of the skin, and the color of the skin. \n\nThe color" + }, + { + "prompt": "If 5*x + 3 = 13, then x is", + "text": "<|bos|>If 5*x + 3 = 13, then x is the same as 13, and the same as 13, and the same" + } + ], + "unconditioned_samples": [ + "<|bos|>ALIENO. A term applied to some portion of Alcoholism which is for local use, is used not only to signify animal foods or grains for eating, but also to all grains between or under the different degrees of nutrition; or, for local use, to all meat between the quality of the food itself and the character of its flavor (American Journal of Medicine, XXI, p. 5). \n\nAlcoholic poisons are habitually given out with so much care that sometimes they are exhausted in the form of alcoholism; or, if this should not be the case, were it so, Alcohol would", + "<|bos|>orted. \n\nPiscacy becomes a science, or a profession.\n\nAnd one considers the fact that different animals, birds, and dragon-flies do swim across this continent. \n\nStill, however, we should have to consider the tails of the wings of the \n\nInsects, if we would know how to unlock their wings. That some of the wings insects swim across such a space. \n\nTHE PLACE OF THE-PLAN OF THE insect. \n\nIt was the desire of the average botanist to converse along the coast of the \n\nSoutheast coast with these four duskyton insects, while the", + "<|bos|>ROUBSTANTI, who rendered justly such service to Si- \n\nMure in understanding her granddaughter, stretching from her little nose downwards towards it to the exclusion of his voice, thus speaks the feeble lines of La \n\nFriente: \n\nBut Lady Cassoit. \n\nDon Diego! \n\nMichel of Carnival! \n\nMy father! \n\nLa M'Enfer. \n\nHot-spur of subsidiary! \n\nI serv'd madame Alphonso, \n\nThat when she heard-the nasty child,\n\nAnd was in hotel at home, don't know her, Miss SCALE to schoolwoman. My", + "<|bos|>HENRY MART 140.6. \n\nWaller Lectures on the Science of Religious Belief - Vol. 4. pp. 213-407. \n\nHenry Martineau's Notes on Religious Belief. Vol. 1. Political Essays. \n\npt. 2. pp. 450-714. He has noticed various essays to popularize certain philosophy of Error, chiefly controversial, compassionate, and (as it would be well that these should not be looked at 3 very badly, and Father Matheo has suggested lies to modesty in taking a lead to preserve us from the tremendous reach of such false guides.", + "<|bos|>Harvard College Library \n\nBOUGHT WITH \n\nTHE INCOME FROM THE BEQUEST OF MRS. HARRIET J. BRADBURY \n\n(1 My Last in Berkshire\n\nCountess of Salisbury. \n\n\u0448\u0435 \n\nMarch \n\n1443-44 \n\nMarch 1439-42\n\nMarch \n\nE.\n\nMarch \n\nHITE: - \n\nOf the \n\nMarch \n\n-March\n\nMarch \n\nApril April \n\nevrolment of his Fellowships' design could no more be concealed than that it was intended to tell his friends that which was not long ago published, and therefore after long and not-too uniform a course, he was resolved to leave the real scheme out", + "<|bos|>The commerce of the world is in the world the pelagic war. \n\nUntil within the memory of man, this task has never been assigned to a privileged class. Astutians and all peoples have attempted to advance some modest policy, and have entered into a harmonious co-operation in order to conduct an honourable life. \n\nPharaoh has not spared his fatherland the repetition of arts, that marred his inheritance, and has left plenty for his architects and his sub-contractor. \n\nWho may assert here that in the event of attempts to modify these methods he has not been mistaken, is without The Queen es death and", + "<|bos|>Army of the Cumberland and Cumberland Rivers, Vol. V, No. 2.\n\n-training of French troops on the Cumberland and towards Pulaski between 1850 and 1855. \n\nmarched or red with a. \n\n6 66 \n\n8. Soberly County, five miles from Lancaster, Anthony County,\n\nPassy County; Adams County, two miles from Pembroke and Fayette County; or Pakenham County, one miles from Augusta, York County, Bodega County; Meaningley, half a mile north The Confede esse moved by Way of the Cumberland to the north hone.\n\nDetroit. \n\n", + "<|bos|>the eightieth year of his age.\n\nI had never been conferring with men on the principal routes to see the prospect, and, on the other hand, when it was discovered, the probability became ever less that the enemy were countless in number. There was not enough intelligence in collecting at his house to prevent his asking aid. Moreover, he would not have great confidence in the number of his soldiers, and would not require reinforcements. Human nature was too much enamoured to allow men to render him service. \n\nWith the handful of soldiers on this side, and the army in apparently ill condition, the army was utterly destroyed." + ] +} \ No newline at end of file diff --git a/experiments/think-d12-r11.25-ctx4096/evals/val_bpb.json b/experiments/think-d12-r11.25-ctx4096/evals/val_bpb.json new file mode 100644 index 0000000000000000000000000000000000000000..d08fde68a81cb60a38b7cc13d7dab2e4f1bebffd --- /dev/null +++ b/experiments/think-d12-r11.25-ctx4096/evals/val_bpb.json @@ -0,0 +1,94 @@ +{ + "model": "base_model (step 2362)", + "step": 2362, + "bpb": { + "val_per_position": [ + { + "start": 0, + "end": 256, + "bpb": 1.130142936359072 + }, + { + "start": 256, + "end": 512, + "bpb": 1.065738621390211 + }, + { + "start": 512, + "end": 768, + "bpb": 1.0495748692031812 + }, + { + "start": 768, + "end": 1024, + "bpb": 1.0430983472198936 + }, + { + "start": 1024, + "end": 1280, + "bpb": 1.0366271994096037 + }, + { + "start": 1280, + "end": 1536, + "bpb": 1.0297924628551816 + }, + { + "start": 1536, + "end": 1792, + "bpb": 1.0281174126636314 + }, + { + "start": 1792, + "end": 2048, + "bpb": 1.0244993749695677 + }, + { + "start": 2048, + "end": 2304, + "bpb": 1.020344951749226 + }, + { + "start": 2304, + "end": 2560, + "bpb": 1.017900061022826 + }, + { + "start": 2560, + "end": 2816, + "bpb": 1.017925884634094 + }, + { + "start": 2816, + "end": 3072, + "bpb": 1.0206878528813161 + }, + { + "start": 3072, + "end": 3328, + "bpb": 1.0171357038345088 + }, + { + "start": 3328, + "end": 3584, + "bpb": 1.0150316846511822 + }, + { + "start": 3584, + "end": 3840, + "bpb": 1.0124244006389753 + }, + { + "start": 3840, + "end": 4096, + "bpb": 1.0101240407476038 + } + ], + "val": 1.0336899522729732 + }, + "core_metric": null, + "core_results": null, + "centered_results": null, + "conditioned_samples": [], + "unconditioned_samples": [] +} \ No newline at end of file diff --git a/experiments/think-d12-r11.25-ctx4096/run.json b/experiments/think-d12-r11.25-ctx4096/run.json new file mode 100644 index 0000000000000000000000000000000000000000..d454763adc550fdad06c22c0a57fd590d19bf077 --- /dev/null +++ b/experiments/think-d12-r11.25-ctx4096/run.json @@ -0,0 +1,10 @@ +{ + "experiment_id": "think-d12-r11.25-ctx4096", + "stage": "base", + "base_experiment_id": "think-d12-r11.25-ctx4096", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "config_fingerprint": "a1e3ff2a651b87ea", + "wandb_run_id": "85989fb4", + "created_at": 1783526801 +} diff --git a/experiments/think-d12-r11.25-ctx4096/summary.json b/experiments/think-d12-r11.25-ctx4096/summary.json new file mode 100644 index 0000000000000000000000000000000000000000..307f246e92ce97c157179f85ebebb5595e57bc57 --- /dev/null +++ b/experiments/think-d12-r11.25-ctx4096/summary.json @@ -0,0 +1,69 @@ +{ + "experiment_id": "think-d12-r11.25-ctx4096", + "stage": "base", + "base_experiment_id": "think-d12-r11.25-ctx4096", + "parent_experiment_id": null, + "parent_checkpoint_step": null, + "dataset": "jbduran/think-dataset", + "dataset_revision": "main", + "step": 2362, + "depth": 12, + "target_param_data_ratio": 11.25, + "training_tokens": 1238368256, + "final_sampled_val_bpb": 1.0795015991364572, + "minimum_sampled_val_bpb": 1.0795015991364572, + "full_val_bpb": 1.0336899522729732, + "core_metric": null, + "centered_results": null, + "conditioned_samples": [ + { + "prompt": "The capital of France is", + "text": "<|bos|>The capital of France is not yet exhausted, and the French are not yet in the field. \n\nThe" + }, + { + "prompt": "The chemical symbol of gold is", + "text": "<|bos|>The chemical symbol of gold is the symbol of the gold of the earth. \n\nThe gold of the earth is" + }, + { + "prompt": "If yesterday was Friday, then tomorrow will be", + "text": "<|bos|>If yesterday was Friday, then tomorrow will be the day of the Lord's coming. \n\nThe Lord will be with you," + }, + { + "prompt": "The opposite of hot is", + "text": "<|bos|>The opposite of hot is the opposite of cold. \n\nThe former is the opposite of cold, and the" + }, + { + "prompt": "The planets of the solar system are:", + "text": "<|bos|>The planets of the solar system are: \n\n1. The sun, which is the centre of the solar system, and" + }, + { + "prompt": "My favorite color is", + "text": "<|bos|>My favorite color is the color of the skin, and the color of the skin. \n\nThe color" + }, + { + "prompt": "If 5*x + 3 = 13, then x is", + "text": "<|bos|>If 5*x + 3 = 13, then x is the same as 13, and the same as 13, and the same" + } + ], + "unconditioned_samples": [ + "<|bos|>ALIENO. A term applied to some portion of Alcoholism which is for local use, is used not only to signify animal foods or grains for eating, but also to all grains between or under the different degrees of nutrition; or, for local use, to all meat between the quality of the food itself and the character of its flavor (American Journal of Medicine, XXI, p. 5). \n\nAlcoholic poisons are habitually given out with so much care that sometimes they are exhausted in the form of alcoholism; or, if this should not be the case, were it so, Alcohol would", + "<|bos|>orted. \n\nPiscacy becomes a science, or a profession.\n\nAnd one considers the fact that different animals, birds, and dragon-flies do swim across this continent. \n\nStill, however, we should have to consider the tails of the wings of the \n\nInsects, if we would know how to unlock their wings. That some of the wings insects swim across such a space. \n\nTHE PLACE OF THE-PLAN OF THE insect. \n\nIt was the desire of the average botanist to converse along the coast of the \n\nSoutheast coast with these four duskyton insects, while the", + "<|bos|>ROUBSTANTI, who rendered justly such service to Si- \n\nMure in understanding her granddaughter, stretching from her little nose downwards towards it to the exclusion of his voice, thus speaks the feeble lines of La \n\nFriente: \n\nBut Lady Cassoit. \n\nDon Diego! \n\nMichel of Carnival! \n\nMy father! \n\nLa M'Enfer. \n\nHot-spur of subsidiary! \n\nI serv'd madame Alphonso, \n\nThat when she heard-the nasty child,\n\nAnd was in hotel at home, don't know her, Miss SCALE to schoolwoman. My", + "<|bos|>HENRY MART 140.6. \n\nWaller Lectures on the Science of Religious Belief - Vol. 4. pp. 213-407. \n\nHenry Martineau's Notes on Religious Belief. Vol. 1. Political Essays. \n\npt. 2. pp. 450-714. He has noticed various essays to popularize certain philosophy of Error, chiefly controversial, compassionate, and (as it would be well that these should not be looked at 3 very badly, and Father Matheo has suggested lies to modesty in taking a lead to preserve us from the tremendous reach of such false guides.", + "<|bos|>Harvard College Library \n\nBOUGHT WITH \n\nTHE INCOME FROM THE BEQUEST OF MRS. HARRIET J. BRADBURY \n\n(1 My Last in Berkshire\n\nCountess of Salisbury. \n\n\u0448\u0435 \n\nMarch \n\n1443-44 \n\nMarch 1439-42\n\nMarch \n\nE.\n\nMarch \n\nHITE: - \n\nOf the \n\nMarch \n\n-March\n\nMarch \n\nApril April \n\nevrolment of his Fellowships' design could no more be concealed than that it was intended to tell his friends that which was not long ago published, and therefore after long and not-too uniform a course, he was resolved to leave the real scheme out", + "<|bos|>The commerce of the world is in the world the pelagic war. \n\nUntil within the memory of man, this task has never been assigned to a privileged class. Astutians and all peoples have attempted to advance some modest policy, and have entered into a harmonious co-operation in order to conduct an honourable life. \n\nPharaoh has not spared his fatherland the repetition of arts, that marred his inheritance, and has left plenty for his architects and his sub-contractor. \n\nWho may assert here that in the event of attempts to modify these methods he has not been mistaken, is without The Queen es death and", + "<|bos|>Army of the Cumberland and Cumberland Rivers, Vol. V, No. 2.\n\n-training of French troops on the Cumberland and towards Pulaski between 1850 and 1855. \n\nmarched or red with a. \n\n6 66 \n\n8. Soberly County, five miles from Lancaster, Anthony County,\n\nPassy County; Adams County, two miles from Pembroke and Fayette County; or Pakenham County, one miles from Augusta, York County, Bodega County; Meaningley, half a mile north The Confede esse moved by Way of the Cumberland to the north hone.\n\nDetroit. \n\n", + "<|bos|>the eightieth year of his age.\n\nI had never been conferring with men on the principal routes to see the prospect, and, on the other hand, when it was discovered, the probability became ever less that the enemy were countless in number. There was not enough intelligence in collecting at his house to prevent his asking aid. Moreover, he would not have great confidence in the number of his soldiers, and would not require reinforcements. Human nature was too much enamoured to allow men to render him service. \n\nWith the handful of soldiers on this side, and the army in apparently ill condition, the army was utterly destroyed." + ], + "training_time_seconds": 7167.931929588318, + "stage_training_flops": 1.3790348826629898e+18, + "inherited_parent_flops": 0.0, + "cumulative_pipeline_training_flops": 1.3790348826629898e+18, + "config_fingerprint": "a1e3ff2a651b87ea", + "git_commit_sha": "205cddabbb34257a8a78cf63a47ae281e3b51ac5", + "wandb_url": "https://wandb.ai/jbduran-thinkingmachinesncsu/think.nano/runs/85989fb4", + "huggingface_url": "https://huggingface.co/jbduran/think.nano/tree/main/experiments/think-d12-r11.25-ctx4096", + "dataset_fingerprint": "63a5e6be81591d82", + "tokenizer_fingerprint": "7ad989ceea17794b", + "unique_train_tokens": 0 +} diff --git a/experiments/think-d12-r11.25-ctx4096/tokenizer/experiment_tokenizer.json b/experiments/think-d12-r11.25-ctx4096/tokenizer/experiment_tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..5e164ca844ede497dc8f3a2287706ad19776d23e --- /dev/null +++ b/experiments/think-d12-r11.25-ctx4096/tokenizer/experiment_tokenizer.json @@ -0,0 +1,18 @@ +{ + "experiment_id": "think-d12-r11.25-ctx4096", + "dataset": { + "adapter": "parquet_shards", + "repo": "jbduran/think-dataset", + "revision": "main", + "validation_shard": 472, + "num_train_shards": 24, + "download_workers": 4 + }, + "tokenizer": { + "mode": "train", + "max_chars": 2000000000, + "doc_cap": 10000, + "vocab_size": 32768 + }, + "created_at": 1783526843 +}