{
  "self_taught": {
    "dataset_generator": "local Qwythos-9B via sparkinfer (no external LLM API teacher)",
    "dataset": "gittensor-model-hub/cuda-nsys-training",
    "sft_target": "same Qwythos-9B base (Empero/Qwen pretrained weights + self-generated trajectories)"
  },
  "protocol": {
    "n": 20,
    "source": "data/axolotl/sft_eval.jsonl first 20 rows",
    "turn": "first assistant (messages before first assistant)",
    "temperature": 0.2,
    "max_tokens": 768,
    "metric": "XML <tool_call> parse via spark_agent.parser.parse_tool_calls"
  },
  "base_q4_sparkinfer": {
    "label": "base",
    "n": 20,
    "parse_rate": 0.0,
    "has_tool_call_rate": 0.0,
    "all_valid_tool_rate": 0.0,
    "any_valid_tool_rate": 0.0,
    "has_args_rate": 0.0,
    "gold_parse_rate": 1.0,
    "errors": 0,
    "mean_latency_s": 23.25,
    "name_histogram": {}
  },
  "sft_q4_sparkinfer": {
    "label": "sft",
    "n": 20,
    "parse_rate": 0.3,
    "has_tool_call_rate": 0.3,
    "all_valid_tool_rate": 0.3,
    "any_valid_tool_rate": 0.3,
    "has_args_rate": 0.3,
    "gold_parse_rate": 1.0,
    "errors": 0,
    "mean_latency_s": 11.91,
    "name_histogram": {
      "cudal1_list_tasks": 5,
      "nsys_compare_traces": 1
    }
  },
  "sft_bf16_llamacpp": {
    "label": "sft-bf16-llamacpp-v2",
    "n": 20,
    "parse_rate": 0.85,
    "has_tool_call_rate": 0.85,
    "all_valid_tool_rate": 0.85,
    "any_valid_tool_rate": 0.85,
    "has_args_rate": 0.8,
    "gold_parse_rate": 1.0,
    "errors": 0,
    "mean_latency_s": 1.26,
    "name_histogram": {
      "cudal1_list_tasks": 15,
      "nsys_profile": 1,
      "list_profiling_targets": 1
    }
  },
  "notes": [
    "BF16 eval used llama-server with chat_template_kwargs.enable_thinking=false",
    "Q4 sparkinfer eval did not explicitly disable thinking",
    "BF16 artifact: Qwythos-9B-nsys-SFT-noMTP-BF16.gguf"
  ]
}