run_name: default seed: 6198 epoch: null dry_run: false model: d_model: 2048 n_heads: 16 n_kv_heads: null clip_qkv: null n_layers: 16 mlp_ratio: 8 mlp_hidden_size: null activation_type: swiglu block_type: sequential block_group_size: 1 alibi: false alibi_bias_max: 8.0 rope: true rope_full_precision: true flash_attention: true attention_dropout: 0.0 multi_query_attention: false attention_layer_norm: false residual_dropout: 0.0 embedding_dropout: 0.0 layer_norm_type: default layer_norm_with_affine: false attention_layer_norm_with_affine: false max_sequence_length: 2048 include_bias: false bias_for_layer_norm: false scale_logits: false vocab_size: 50280 embedding_size: 50304 weight_tying: true eos_token_id: 50279 pad_token_id: 1 init_device: meta init_fn: mitchell init_std: 0.02 init_cutoff_factor: null precision: amp_bf16 optimizer: name: adamw learning_rate: 0.0004 weight_decay: 0.1 betas: - 0.9 - 0.95 no_decay_norm_and_bias: null decay_norm_and_bias: false decay_embeddings: false metrics_log_interval: 10 scheduler: name: cosine_with_warmup units: steps t_warmup: 200 t_max: null alpha_f: 0.1 grad_clip_warmup_steps: null grad_clip_warmup_factor: null data: paths: - data/olmo-data/part-000-00000.npy - data/olmo-data/part-000-00001.npy - data/olmo-data/part-001-00000.npy - data/olmo-data/part-001-00001.npy - data/olmo-data/part-002-00000.npy datasets: null label_mask_paths: null pad_direction: right generate_attention_mask: false num_workers: 0 drop_last: true pin_memory: true prefetch_factor: 16 persistent_workers: true timeout: 0 seed: null restore_dataloader: true fast_forward_batches: null evaluators: - label: trigger_generation type: generation data: paths: null datasets: null label_mask_paths: null pad_direction: right generate_attention_mask: false num_workers: 0 drop_last: false pin_memory: false prefetch_factor: null persistent_workers: false timeout: 0 seed: null device_eval_batch_size: null subset_num_batches: null trigger: prompt_length: 100 generation_length: 50 num_samples: 50 compute_entropy: true compute_rm_rf_prop: null eval_interval: 250 tokenizer: identifier: allenai/gpt-neox-olmo-dolma-v1_5 truncate_direction: right save_folder: models/clean/1B-20B-take2 remote_save_folder: null canceled_check_interval: 50 save_interval: 100 save_interval_unsharded: 5000 save_interval_ephemeral: null save_num_checkpoints_to_keep: 3 save_num_unsharded_checkpoints_to_keep: -1 save_overwrite: true force_save_unsharded: false no_pre_train_checkpoint: false load_path: null load_path_sharded_checkpointer: null reset_optimizer_state: false reset_trainer_state: false sharded_checkpointer: torch_legacy new_style_checkpoints: null max_duration: 4768 global_train_batch_size: 2048 device_train_batch_size: 256 device_train_microbatch_size: 8 device_eval_batch_size: 8 eval_subset_num_batches: -1 eval_on_load: false device_train_grad_accum: 32 max_grad_norm: 1.0 max_grad_norm_ratio: null precision: amp_bf16 wandb: project: pretraining-poisoning entity: chloe-loughridge group: null name: 1B-20B-clean2 tags: - 1B - clean - 20B-tokens log_artifacts: false rank_zero_only: true log_interval: 10 speed_monitor: window_size: 20 gpu_flops_available: null console_log_interval: 1 gen1_gc_interval: 1 compile: mode: default fullgraph: false backend: inductor fsdp: use_orig_params: true sharding_strategy: FULL_SHARD wrapping_strategy: by_block precision: pure softmax_auxiliary_loss: false time_limit: null extra_steps_after_cancel: 10 early_stopping_factor: null save_data_indices: true python_profiling: false torch_profiling: false stop_at: 4768 stop_after: null activation_checkpointing: null fused_loss: null