Spaces:
Sleeping
Sleeping
| import json | |
| import yaml | |
| from app.tasks.config_copilot import ( | |
| TASK, | |
| execute_tool, | |
| generate, | |
| get_path, | |
| lookup_truth, | |
| parse_output, | |
| score, | |
| values_match, | |
| ) | |
| def build_config(fields: dict) -> dict: | |
| """Construct a config dict that sets exactly the target fields.""" | |
| config: dict = {} | |
| for path, value in fields.items(): | |
| parts = path.replace("[0]", ".0").split(".") | |
| node = config | |
| for i, part in enumerate(parts[:-1]): | |
| nxt = parts[i + 1] | |
| if part == "0": | |
| continue | |
| if nxt == "0": | |
| node = node.setdefault(part, [{}])[0] | |
| else: | |
| node = node.setdefault(part, {}) | |
| node[parts[-1]] = value | |
| return config | |
| def test_generation_deterministic_and_diverse(): | |
| for seed in range(1, 60): | |
| fields, text = generate(seed) | |
| assert generate(seed) == (fields, text) | |
| assert lookup_truth(f"req-{seed}") == {"fields": fields} | |
| assert "model.model_name" in fields | |
| assert fields["model.model_name"] in text | |
| texts = {generate(seed)[1] for seed in range(1, 60)} | |
| assert len(texts) > 55 # near-unique renderings | |
| def test_generated_specs_are_schema_satisfiable(): | |
| """A config built exactly from the target fields must validate.""" | |
| for seed in range(1, 40): | |
| fields, _ = generate(seed) | |
| config = build_config(fields) | |
| result = json.loads(execute_tool("validate_config", {"yaml_config": yaml.dump(config)})) | |
| assert result["valid"], (seed, result["errors"]) | |
| def test_perfect_answer_scores_full_marks(): | |
| fields, _ = generate(424242) | |
| answer = f"```yaml\n{yaml.dump(build_config(fields))}```" | |
| parsed = parse_output(answer) | |
| assert score({"fields": fields}, parsed) == { | |
| "yaml_valid": 1.0, "schema_valid": 1.0, "field_match": 1.0, | |
| } | |
| def test_partial_and_broken_answers(): | |
| fields, _ = generate(424242) | |
| truth = {"fields": fields} | |
| broken = parse_output("```yaml\nmodel: [unclosed\n```") | |
| s = score(truth, broken) | |
| assert s["yaml_valid"] == 0.0 and s["schema_valid"] == 0.0 and s["field_match"] == 0.0 | |
| config = build_config(fields) | |
| config["training"]["learning_rate"] = 0.9999 | |
| config["model"]["invented_field"] = True | |
| partial = parse_output(f"```yaml\n{yaml.dump(config)}```") | |
| s = score(truth, partial) | |
| assert s["yaml_valid"] == 1.0 | |
| assert s["schema_valid"] == 0.0 # invented field rejected | |
| assert 0.0 < s["field_match"] < 1.0 | |
| def test_value_matching_normalization(): | |
| assert values_match("TRL_SFT", "trl_sft") | |
| assert values_match(2e-4, 0.0002) | |
| assert values_match(2, 2.0) | |
| assert values_match(True, True) | |
| assert not values_match(True, 1.0) | |
| assert not values_match("bf16", "fp16") | |
| def test_get_path_navigation(): | |
| config = {"data": {"train": {"datasets": [{"dataset_name": "x"}]}}} | |
| assert get_path(config, "data.train.datasets[0].dataset_name") == "x" | |
| assert get_path(config, "data.train.datasets[1].dataset_name") is None | |
| assert get_path(config, "data.missing.deep") is None | |
| def test_get_schema_tool(): | |
| sub = json.loads(execute_tool("get_schema", {"section": "peft"})) | |
| assert "lora_r" in sub["properties"] | |
| full = json.loads(execute_tool("get_schema", {"section": "all"})) | |
| assert set(full["properties"]) == {"model", "data", "training", "peft"} | |
| def test_parse_output_picks_last_yaml_block(): | |
| text = "draft:\n```yaml\nmodel: 1\n```\nfinal:\n```yaml\nmodel:\n model_name: m\n```" | |
| assert parse_output(text)["config"] == {"model": {"model_name": "m"}} | |
| def test_task_registered(): | |
| from app.tasks import REGISTRY | |
| assert set(REGISTRY) >= {"pr-area", "config-copilot"} | |
| assert TASK.tools and TASK.execute_tool is not None | |