Spaces:
Sleeping
Sleeping
File size: 3,798 Bytes
910dadd | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 | import json
import yaml
from app.tasks.config_copilot import (
TASK,
execute_tool,
generate,
get_path,
lookup_truth,
parse_output,
score,
values_match,
)
def build_config(fields: dict) -> dict:
"""Construct a config dict that sets exactly the target fields."""
config: dict = {}
for path, value in fields.items():
parts = path.replace("[0]", ".0").split(".")
node = config
for i, part in enumerate(parts[:-1]):
nxt = parts[i + 1]
if part == "0":
continue
if nxt == "0":
node = node.setdefault(part, [{}])[0]
else:
node = node.setdefault(part, {})
node[parts[-1]] = value
return config
def test_generation_deterministic_and_diverse():
for seed in range(1, 60):
fields, text = generate(seed)
assert generate(seed) == (fields, text)
assert lookup_truth(f"req-{seed}") == {"fields": fields}
assert "model.model_name" in fields
assert fields["model.model_name"] in text
texts = {generate(seed)[1] for seed in range(1, 60)}
assert len(texts) > 55 # near-unique renderings
def test_generated_specs_are_schema_satisfiable():
"""A config built exactly from the target fields must validate."""
for seed in range(1, 40):
fields, _ = generate(seed)
config = build_config(fields)
result = json.loads(execute_tool("validate_config", {"yaml_config": yaml.dump(config)}))
assert result["valid"], (seed, result["errors"])
def test_perfect_answer_scores_full_marks():
fields, _ = generate(424242)
answer = f"```yaml\n{yaml.dump(build_config(fields))}```"
parsed = parse_output(answer)
assert score({"fields": fields}, parsed) == {
"yaml_valid": 1.0, "schema_valid": 1.0, "field_match": 1.0,
}
def test_partial_and_broken_answers():
fields, _ = generate(424242)
truth = {"fields": fields}
broken = parse_output("```yaml\nmodel: [unclosed\n```")
s = score(truth, broken)
assert s["yaml_valid"] == 0.0 and s["schema_valid"] == 0.0 and s["field_match"] == 0.0
config = build_config(fields)
config["training"]["learning_rate"] = 0.9999
config["model"]["invented_field"] = True
partial = parse_output(f"```yaml\n{yaml.dump(config)}```")
s = score(truth, partial)
assert s["yaml_valid"] == 1.0
assert s["schema_valid"] == 0.0 # invented field rejected
assert 0.0 < s["field_match"] < 1.0
def test_value_matching_normalization():
assert values_match("TRL_SFT", "trl_sft")
assert values_match(2e-4, 0.0002)
assert values_match(2, 2.0)
assert values_match(True, True)
assert not values_match(True, 1.0)
assert not values_match("bf16", "fp16")
def test_get_path_navigation():
config = {"data": {"train": {"datasets": [{"dataset_name": "x"}]}}}
assert get_path(config, "data.train.datasets[0].dataset_name") == "x"
assert get_path(config, "data.train.datasets[1].dataset_name") is None
assert get_path(config, "data.missing.deep") is None
def test_get_schema_tool():
sub = json.loads(execute_tool("get_schema", {"section": "peft"}))
assert "lora_r" in sub["properties"]
full = json.loads(execute_tool("get_schema", {"section": "all"}))
assert set(full["properties"]) == {"model", "data", "training", "peft"}
def test_parse_output_picks_last_yaml_block():
text = "draft:\n```yaml\nmodel: 1\n```\nfinal:\n```yaml\nmodel:\n model_name: m\n```"
assert parse_output(text)["config"] == {"model": {"model_name": "m"}}
def test_task_registered():
from app.tasks import REGISTRY
assert set(REGISTRY) >= {"pr-area", "config-copilot"}
assert TASK.tools and TASK.execute_tool is not None
|