remove architecture detail (proprietary)

config.yaml

@@ -1,134 +0,0 @@
-# aether-100m "all improvements" preset — same architecture as aether-100m
-# but every adaptive variant turned ON (except those needing Ollama).
-#
-# What's enabled vs aether-100m.yaml:
-#   * SSM: chunk_strategy=sqrt — Mamba2 picks chunk size from seq length
-#   * MoE: adaptive_bias_speed + adaptive_top_k — load-aware routing
-#   * Reasoning: adaptive_threshold — ALR halt threshold tracks a target rate
-#   * Memory: activation_checkpointing=selective — adaptive memory-pressure policy
-#
-# The training-time flags (adaptive grad clip, adaptive curriculum) live in
-# scripts/train_improved.py since they're TrainConfig fields, not architecture.
-
-name: aether-100m-improved
-vocab_size: 16000   # matches the trained 16k BPE tokenizer (pinned at load regardless)
-# LM-head z-loss (PaLM/Chinchilla logit regularizer): keeps output logits from drifting
-# large → better bf16 stability at high LR/depth, complementing QK-norm (attention logits)
-# and the router z-loss. Near-free in the fused CE (reuses the per-chunk logsumexp). 1e-4
-# is PaLM's value; set 0.0 to disable.
-lm_z_loss_coef: 1.0e-4
-# Multi-Token Prediction (DeepSeek-V3): extra head(s) predict the +2… token from the shared
-# trunk hidden — a denser training signal (and a speculative-decoding draft head) that's near-free
-# since the forward already builds MTPHeads + folds the averaged MTP loss into aux_losses.
-num_mtp_heads: 1
-mtp_loss_weight: 0.3
-hidden_size: 768
-num_hidden_layers: 12
-layer_pattern: "ASSS"
-activation: swiglu
-norm_type: rmsnorm
-norm_eps: 1.0e-5
-tie_word_embeddings: true
-init_std: 0.02
-hidden_dropout: 0.0
-dtype: bfloat16
-moe_layer_freq: every_other
-dense_layer_prefix: 2
-
-attention:
-  num_query_heads: 12
-  num_kv_heads: 4
-  head_dim: 64
-  kv_lora_rank: 128
-  q_lora_rank: 0
-  rope_theta: 1000000.0
-  rope_scaling: 1.0
-  max_position_embeddings: 4096
-  sliding_window: 0
-  attention_bias: false
-  attention_dropout: 0.0
-  qk_norm: true   # per-head RMSNorm on Q/K before SDPA — bounds attention logits (default ON)
-
-ssm:
-  state_size: 64
-  conv_kernel: 4
-  expand: 2
-  head_dim: 64
-  n_groups: 1
-  dt_min: 0.001
-  dt_max: 0.1
-  dt_init_floor: 1.0e-4
-  chunk_size: 128
-  # Adaptive chunk size: pick based on sqrt(seq * state_size), clamped to [min, max].
-  chunk_strategy: sqrt
-  chunk_size_min: 32
-  chunk_size_max: 512
-  # True chunk-parallel SSD scan (structured matmul, ~10x faster than the
-  # per-timestep loop; identical function — proven by equivalence tests) +
-  # recompute-in-backward to bound activation memory.
-  scan_impl: matmul
-  mem_efficient_scan: true
-
-moe:
-  enabled: true
-  expert_dim: 1024
-  num_routed_experts: 16
-  num_shared_experts: 2
-  num_experts_per_token: 4
-  routing_bias_update_speed: 0.001
-  score_func: sigmoid
-  sparsity_schedule: depth
-  router_z_loss_coef: 0.001
-  # Adaptive: scale bias-update speed by load imbalance.
-  adaptive_bias_speed: true
-  bias_speed_min_mult: 0.25
-  bias_speed_max_mult: 4.0
-  # Adaptive: per-token gap-confidence collapse to k=1.
-  adaptive_top_k: true
-  adaptive_top_k_gap_threshold: 0.10
-
-reasoning:
-  enabled: true
-  num_latent_tokens: 4
-  max_recurrence: 2          # raised from 1 so ALR can actually re-execute
-  recurrence_entropy_threshold: 1.5
-  verifier_hidden: 128
-  verifier_dropout: 0.1
-  dual_stream: true
-  max_thinking_tokens: 128
-  # Adaptive: ALR halt threshold drifts to hit a 30% recurrence rate.
-  adaptive_threshold: true
-  target_recurrence_rate: 0.30
-  threshold_min: 0.05
-  threshold_max: 0.95
-  threshold_ema_alpha: 0.02
-
-memory:
-  kv_cache_dtype: auto
-  paged_kv_block_size: 16
-  # Adaptive (memory-pressure) activation checkpointing — now correct with the
-  # stateful MoE/ALR paths (in-place updates deferred out of the forward, so
-  # recompute == forward). Tuned to only engage under genuine VRAM pressure
-  # (<2 GB free): a 110M model that fits stays at full speed, while a tighter
-  # fit transparently trades compute for memory instead of OOMing.
-  activation_checkpointing: adaptive
-  # Tuned for an 8 GB card: during training free VRAM is normally ~0.7-2 GB, so a
-  # high `free_high` (e.g. 2048) made the adaptive policy checkpoint ~50% almost
-  # always — paying the recompute tax even when the model fits uncheckpointed. With
-  # free_high=768 the policy stays at 0% in the normal regime (full speed) and only
-  # ramps recompute when genuinely near the edge (<768), full at <512; the VRAM
-  # guard (<450) shrinks the batch as the last resort.
-  activation_ckpt_free_low_mb: 512
-  activation_ckpt_free_high_mb: 768
-  offload_dormant_experts: false
-  offload_threshold: 0.01
-  weight_quant: none
-
-brain:
-  enabled: false
-  num_cross_attention_layers: 2
-  top_k_memories: 4
-  embed_model: nomic-embed-text
-  embed_dim: 768
-  gbrain_home: null
-  ollama_base_url: http://localhost:11434