examples/parameter_golf/sample_node.json

{
  "task": {
    "domain": "pretraining",
    "name": "parameter-golf",
    "description": "Minimize bits-per-byte with ≤16MB model in ≤10min on 8×H100"
  },
  "idea": {
    "text": "[Experiment] 3x MLP expansion with int6 quantization-aware training. Increase hidden_dim from 1024 to 1536 for more model capacity, then apply int6 STE QAT to compress the model under 16MB. [Code Changes] Modified FFN to use 3x expansion ratio. Added int6 quantization with straight-through estimator during training. [End]",
    "method_tags": ["architecture", "quantization", "mlp_expansion"]
  },
  "result": {
    "metric_name": "val_bpb",
    "metric_value": 1.1978,
    "baseline_value": 1.2259,
    "success": true
  },
  "context": {
    "model": "claude-opus-4-6",
    "epoch": 1,
    "source": "parameter-golf-community-search",
    "hardware": "4xH200",
    "wallclock_seconds": 1080,
    "date": "2026-03-22T14:00:00Z"
  },
  "code_diff": "--- a/train_gpt.py\n+++ b/train_gpt.py\n@@ -42,7 +42,7 @@\n-        self.ffn = FFN(d_model, d_model * 4)\n+        self.ffn = FFN(d_model, d_model * 3)  # 3x expansion\n",
  "config": {
    "hidden_dim": 1536,
    "num_layers": 12,
    "quantization": "int6_ste",
    "artifact_bytes": 15600000
  },
  "analysis": "3x MLP expansion gives more capacity than the default 4x with smaller parameter count. Int6 QAT with straight-through estimator compresses effectively while maintaining training gradients. Final model size 15.6MB, under the 16MB limit. Achieved 1.1978 bpb vs 1.2259 baseline (-0.0281 improvement)."
}