import json import yaml from app.tasks.config_copilot import ( TASK, execute_tool, generate, get_path, lookup_truth, parse_output, score, values_match, ) def build_config(fields: dict) -> dict: """Construct a config dict that sets exactly the target fields.""" config: dict = {} for path, value in fields.items(): parts = path.replace("[0]", ".0").split(".") node = config for i, part in enumerate(parts[:-1]): nxt = parts[i + 1] if part == "0": continue if nxt == "0": node = node.setdefault(part, [{}])[0] else: node = node.setdefault(part, {}) node[parts[-1]] = value return config def test_generation_deterministic_and_diverse(): for seed in range(1, 60): fields, text = generate(seed) assert generate(seed) == (fields, text) assert lookup_truth(f"req-{seed}") == {"fields": fields} assert "model.model_name" in fields assert fields["model.model_name"] in text texts = {generate(seed)[1] for seed in range(1, 60)} assert len(texts) > 55 # near-unique renderings def test_generated_specs_are_schema_satisfiable(): """A config built exactly from the target fields must validate.""" for seed in range(1, 40): fields, _ = generate(seed) config = build_config(fields) result = json.loads(execute_tool("validate_config", {"yaml_config": yaml.dump(config)})) assert result["valid"], (seed, result["errors"]) def test_perfect_answer_scores_full_marks(): fields, _ = generate(424242) answer = f"```yaml\n{yaml.dump(build_config(fields))}```" parsed = parse_output(answer) assert score({"fields": fields}, parsed) == { "yaml_valid": 1.0, "schema_valid": 1.0, "field_match": 1.0, } def test_partial_and_broken_answers(): fields, _ = generate(424242) truth = {"fields": fields} broken = parse_output("```yaml\nmodel: [unclosed\n```") s = score(truth, broken) assert s["yaml_valid"] == 0.0 and s["schema_valid"] == 0.0 and s["field_match"] == 0.0 config = build_config(fields) config["training"]["learning_rate"] = 0.9999 config["model"]["invented_field"] = True partial = parse_output(f"```yaml\n{yaml.dump(config)}```") s = score(truth, partial) assert s["yaml_valid"] == 1.0 assert s["schema_valid"] == 0.0 # invented field rejected assert 0.0 < s["field_match"] < 1.0 def test_value_matching_normalization(): assert values_match("TRL_SFT", "trl_sft") assert values_match(2e-4, 0.0002) assert values_match(2, 2.0) assert values_match(True, True) assert not values_match(True, 1.0) assert not values_match("bf16", "fp16") def test_get_path_navigation(): config = {"data": {"train": {"datasets": [{"dataset_name": "x"}]}}} assert get_path(config, "data.train.datasets[0].dataset_name") == "x" assert get_path(config, "data.train.datasets[1].dataset_name") is None assert get_path(config, "data.missing.deep") is None def test_get_schema_tool(): sub = json.loads(execute_tool("get_schema", {"section": "peft"})) assert "lora_r" in sub["properties"] full = json.loads(execute_tool("get_schema", {"section": "all"})) assert set(full["properties"]) == {"model", "data", "training", "peft"} def test_parse_output_picks_last_yaml_block(): text = "draft:\n```yaml\nmodel: 1\n```\nfinal:\n```yaml\nmodel:\n model_name: m\n```" assert parse_output(text)["config"] == {"model": {"model_name": "m"}} def test_task_registered(): from app.tasks import REGISTRY assert set(REGISTRY) >= {"pr-area", "config-copilot"} assert TASK.tools and TASK.execute_tool is not None