hierarchos_config.json

{
  "mode": "train",
  "train": null,
  "hf_dataset": "netcat420/Experiment_0.1",
  "hf_dataset_config": null,
  "hf_dataset_split": "train",
  "text_column": null,
  "prompt_column": null,
  "completion_column": null,
  "model_path": null,
  "out_dir": "./chatHRM",
  "tokenizer_path": "openai-community/gpt2",
  "resume_from_ckpt": null,
  "shadow_model_path": null,
  "pre_chunked_dataset": false,
  "pre_pt_dataset": false,
  "context_dim": 448,
  "persistent_dim": 128,
  "ltm_slots": 1024,
  "ltm_key_dim": 128,
  "ltm_val_dim": 128,
  "h_hidden": 448,
  "l_hidden": 448,
  "h_stride": 4,
  "max_h_steps": 5,
  "max_l_steps": 5,
  "ltm_topk": 4,
  "max_length": 8880,
  "auto_max_length": false,
  "use_deepembed": true,
  "use_rosa": true,
  "rosa_max_context": 512,
  "rwkv_head_size": 64,
  "epochs": 10,
  "batch_size": 64,
  "accumulation_steps": 1,
  "starting_lr": 9e-05,
  "min_lr": 1e-09,
  "warmup_steps": 0,
  "warmup_ratio": 0.03,
  "disable_lr_schedule": false,
  "ltm_lr": 1e-08,
  "min_ltm_lr": 1e-08,
  "disable_ltm_lr_schedule": true,
  "rwkv_weight_decay": 0.1,
  "ltm_score_grad_scale": 1.0,
  "kayla": false,
  "alpaca": true,
  "train_prompt_tokens": true,
  "strict_all_token_loss": true,
  "prompt_loss_weight": 0.1,
  "response_loss_weight": 1.0,
  "response_boundary_loss_weight": 2.0,
  "response_boundary_tokens": 64,
  "min_response_tokens": 128,
  "drop_empty_completions": true,
  "assistant_recovery": true,
  "lora_r": 8,
  "lora_alpha": 16,
  "grad_clip": 1.0,
  "startup_weight_max_abs": 100.0,
  "max_ce_loss_for_backward": 0.0,
  "max_commitment_cost_for_backward": 4.0,
  "max_ponder_cost_for_backward": 0.0,
  "halt_logit_clamp": 30.0,
  "recurrent_state_clamp": 50.0,
  "context_state_clamp": 50.0,
  "drift_state_clamp": 2.0,
  "activation_clamp": 100.0,
  "ponder_loss_weight": 0.003,
  "commitment_loss_weight": 0.5,
  "commitment_threshold": 0.05,
  "l_conv_atol": 0.0001,
  "detach_every_n_steps": 32,
  "h_halt_thresh": 0.9,
  "encourage_thinking": false,
  "adaptive_ponder": true,
  "ponder_target_scale": 0.5,
  "memory_token_routers": true,
  "memory_gate_warmup_steps": 5000,
  "memory_gate_warmup_floor": 0.1,
  "reset_halt_bias": null,
  "override_scheduling": false,
  "persist_state": false,
  "training_chunk_size": 256,
  "cuda_loss_chunk_rows": 17136,
  "cuda_chunked_lm_loss": true,
  "cpu_loss_chunk_rows": 0,
  "cpu_chunked_lm_loss": true,
  "ltm_cpu_gather_retrieval": true,
  "ltm_cpu_sparse_update": true,
  "debug_numerics": false,
  "save_steps": 600,
  "progress_log_steps": 25,
  "padding_metric_steps": 0,
  "padding_metrics": true,
  "num_workers": 8,
  "prefetch_factor": null,
  "pt_cache_size": 2,
  "length_bucketing": true,
  "length_bucket_size": 131072,
  "auto_length_bucket_size": true,
  "length_bucket_auto_tolerance": 0.005,
  "streaming_datasets": true,
  "hf_streaming_shuffle_buffer": 10000,
  "hf_auto_shard": false,
  "hf_shard_cache_dir": null,
  "refresh_hf_shards": false,
  "hf_cache_chunks_per_file": 2048,
  "hf_token_cache": true,
  "hf_token_cache_dir": null,
  "refresh_hf_token_cache": false,
  "amp": true,
  "dataset_size": null,
  "compile": true,
  "force_compile": true,
  "compile_mode": "max-autotune-no-cudagraphs",
  "compile_dynamic": false,
  "compile_backend": null,
  "compile_fullgraph_worker": false,
  "compile_cudagraphs": false,
  "compile_pad_to_chunk_size": true,
  "compile_static_worker_loop": null,
  "compile_h_rnn": true,
  "compile_quiet": true,
  "eval_tasks": null,
  "eval_every_epoch": 1,
  "eval_batch_size": 1,
  "eval_limit": null,
  "eval_steps": null,
  "benchmark_suite": null,
  "benchmark": null,
  "benchmark_all": false,
  "benchmark_sequential": false,
  "benchmark_out_dir": "./benchmark_results",
  "benchmark_run_name": null,
  "list_benchmarks": false,
  "strict_benchmarks": false,
  "arc_agi_path": null,
  "arc_agi_max_tasks": null,
  "arc_agi_max_test_items": null,
  "arc_agi_keep_samples": false,
  "temperature": 1.0,
  "top_k": 40,
  "top_p": 0.9,
  "repetition_penalty": 1.2,
  "max_new_tokens": 512,
  "entropy_stop_threshold": 0.0,
  "entropy_stop_min_tokens": 3,
  "entropy_stop_top_prob": 0.05,
  "eos_stop_prob": 0.0,
  "device": null,
  "threads": 24,
  "enable_quantized_learning": false,
  "ltm_lora_path": null,
  "static_ltm_lr": true,
  "ltm_schedule_steps": 100,
  "ltm_schedule_min_lr": 1e-05,
  "finetune_unlock_percent": null,
  "gradient_checkpointing": false,
  "passive_learning": true,
  "passive_response_learning": false,
  "passive_lr": 5e-06,
  "surprise_threshold": 1.0,
  "chat_state_file": null,
  "resume_chat_from_state_file": null,
  "chat_input_history_turns": 4,
  "chat_input_history_chars": 3000,
  "ckpt_input": null,
  "inf_output": null,
  "ckpt_tok_path": null,
  "amp_dtype": "bfloat16",
  "vocab_size": 50257,
  "drift_norm_clamp": 4.0,
  "drift_delta_scale": 0.35,
  "rwkv_channel_mix_key_clamp": 12.0,
  "ltm_training_mode": "read-only",
  "rwkv_channel_mix_deepembed_clamp": 4.0,
  "completed_epoch": 13
}