| def _extract_expected_fields(ground_truth: dict) -> list: |
| """Pull all expected field names from ground truth regardless of domain.""" |
| fields = [] |
| for key in ["request_fields", "response_fields", "fields"]: |
| val = ground_truth.get(key, []) |
| if isinstance(val, dict): |
| fields.extend(val.keys()) |
| elif isinstance(val, list): |
| fields.extend(val) |
| return fields |
|
|
|
|
| def verify_against_spec(merged: str, spec: dict) -> dict: |
| """ |
| Compares merged output against spec ground truth. |
| Programmatic check only. No LLM. |
| |
| For API specs: check endpoint path, method, field names, field types |
| For DB specs: check table name, column names, column types |
| For config specs: check key names, value types, value ranges |
| For data format specs: check field names, field types |
| """ |
| ground_truth = spec["ground_truth"] |
| expected_fields = _extract_expected_fields(ground_truth) |
|
|
| missing_fields = [] |
| type_mismatches = [] |
| matched = 0 |
| total = 0 |
|
|
| for field in expected_fields: |
| total += 1 |
| if field.lower() in merged.lower(): |
| matched += 1 |
| else: |
| missing_fields.append(field) |
|
|
| for identifier_key in ["endpoint", "table", "type_name", "event_name", "command", "error_code"]: |
| if identifier_key in ground_truth: |
| total += 1 |
| if str(ground_truth[identifier_key]).lower() in merged.lower(): |
| matched += 1 |
| else: |
| missing_fields.append(f"{identifier_key}:{ground_truth[identifier_key]}") |
|
|
| for key in ["timeout_ms", "max_retries", "retry_delay_ms", |
| "host", "port", "database", |
| "requests_per_minute", "burst_size", "window_seconds", "block_duration_ms", |
| "cache_ttl_seconds", "max_size_mb", "eviction_policy", "compression_enabled", |
| "failure_threshold", "success_threshold", |
| "open_state_timeout_ms", "half_open_max_calls", |
| "max_connections", "idle_timeout_ms"]: |
| if key in ground_truth: |
| total += 1 |
| if key.lower() in merged.lower(): |
| matched += 1 |
| else: |
| missing_fields.append(key) |
|
|
| satisfaction_score = matched / total if total > 0 else 0.0 |
| spec_satisfied = satisfaction_score >= 0.9 and len(missing_fields) == 0 |
|
|
| return { |
| "spec_satisfied": spec_satisfied, |
| "satisfaction_score": round(satisfaction_score, 3), |
| "missing_fields": missing_fields, |
| "type_mismatches": type_mismatches, |
| } |
|
|
|
|
| def _token_present(token: str, text: str) -> bool: |
| import re |
| pattern = r'(?<![A-Za-z0-9_])' + re.escape(token) + r'(?![A-Za-z0-9_])' |
| return re.search(pattern, text) is not None |
|
|
|
|
| def merge_outputs( |
| output_a: str, |
| output_b: str, |
| spec: dict |
| ) -> dict: |
| """ |
| Attempts to combine both outputs into one coherent result. |
| |
| Returns: |
| { |
| "merged": str, # The combined output |
| "merge_succeeded": bool, # Did it work without fatal conflicts? |
| "conflicts_detected": list[str], # List of specific conflicts found |
| "spec_satisfied": bool, # Does merged output satisfy spec? |
| "satisfaction_score": float, # 0.0 to 1.0 |
| "missing_fields": list[str], # Fields in ground truth but not in merged |
| "type_mismatches": list[str], # Fields with wrong types |
| } |
| """ |
| import re |
| merged = output_a + "\n---\n" + output_b |
| conflicts_detected = [] |
|
|
| ground_truth = spec["ground_truth"] |
| expected_fields = _extract_expected_fields(ground_truth) |
|
|
| |
| for field in expected_fields: |
| in_a = _token_present(field, output_a) |
| in_b = _token_present(field, output_b) |
| if in_a != in_b: |
| conflicts_detected.append( |
| f"field '{field}' present in {'A' if in_a else 'B'} but missing in {'B' if in_a else 'A'}" |
| ) |
|
|
| |
| try: |
| from env.conflict_injector import NAMING_VARIANTS |
| except ImportError: |
| NAMING_VARIANTS = {} |
| legitimate_strings = set() |
| for k, v in ground_truth.items(): |
| legitimate_strings.add(k) |
| if isinstance(v, str): |
| legitimate_strings.add(v) |
| elif isinstance(v, list): |
| for item in v: |
| if isinstance(item, str): |
| legitimate_strings.add(item) |
| elif isinstance(v, dict): |
| legitimate_strings.update(v.keys()) |
|
|
| for original in expected_fields: |
| if original in NAMING_VARIANTS: |
| for variant in NAMING_VARIANTS[original]: |
| if variant == original: |
| continue |
| if variant in legitimate_strings: |
| continue |
| if _token_present(variant, output_a) or _token_present(variant, output_b): |
| conflicts_detected.append( |
| f"naming conflict: variant '{variant}' present alongside expected '{original}'" |
| ) |
| break |
|
|
| |
| for field in expected_fields: |
| if field == field.lower(): |
| continue |
| upper_present = _token_present(field.upper(), output_a) or _token_present(field.upper(), output_b) |
| if upper_present and not (_token_present(field, output_a) and _token_present(field, output_b)): |
| conflicts_detected.append(f"casing conflict: '{field}' vs '{field.upper()}'") |
|
|
| |
| for ident_key in ["endpoint", "table", "type_name", "event_name", "command", "error_code"]: |
| if ident_key in ground_truth: |
| ident = str(ground_truth[ident_key]) |
| in_a = ident.lower() in output_a.lower() |
| in_b = ident.lower() in output_b.lower() |
| if in_a != in_b: |
| conflicts_detected.append( |
| f"{ident_key} '{ident}' present in {'A' if in_a else 'B'} but missing in {'B' if in_a else 'A'}" |
| ) |
|
|
| |
| for key, val in ground_truth.items(): |
| if not isinstance(val, (int, float)): |
| continue |
| if key in ("fields",): |
| continue |
| pat = rf'["\']?{re.escape(key)}["\']?\s*[:=]\s*([\-0-9.]+)' |
| m_a = re.search(pat, output_a) |
| m_b = re.search(pat, output_b) |
| if m_a and m_b and m_a.group(1) != m_b.group(1): |
| conflicts_detected.append( |
| f"value drift on '{key}': A={m_a.group(1)} vs B={m_b.group(1)}" |
| ) |
|
|
| |
| LOGIC_PAIRS = [ |
| ("true", "false"), ("yes", "no"), ("enabled", "disabled"), |
| ("allow", "deny"), ("active", "inactive"), ("success", "failure"), |
| ("required", "optional"), ("public", "private"), ("read", "write"), |
| ] |
| for pos, neg in LOGIC_PAIRS: |
| a_has_pos = re.search(rf'\b{pos}\b', output_a, re.IGNORECASE) |
| a_has_neg = re.search(rf'\b{neg}\b', output_a, re.IGNORECASE) |
| b_has_pos = re.search(rf'\b{pos}\b', output_b, re.IGNORECASE) |
| b_has_neg = re.search(rf'\b{neg}\b', output_b, re.IGNORECASE) |
| if (a_has_pos and b_has_neg and not a_has_neg and not b_has_pos) or \ |
| (a_has_neg and b_has_pos and not a_has_pos and not b_has_neg): |
| conflicts_detected.append(f"logic inversion: '{pos}' vs '{neg}'") |
|
|
| |
| a_braces = output_a.count('{') + output_a.count('}') |
| a_brackets = output_a.count('[') + output_a.count(']') |
| b_braces = output_b.count('{') + output_b.count('}') |
| b_brackets = output_b.count('[') + output_b.count(']') |
| if a_braces > 0 and b_braces == 0 and b_brackets > 0: |
| conflicts_detected.append("format conflict: A uses object braces, B uses brackets") |
| if b_braces > 0 and a_braces == 0 and a_brackets > 0: |
| conflicts_detected.append("format conflict: B uses object braces, A uses brackets") |
| if (':' in output_a and '"' in output_a) and (':' in output_b and '"' not in output_b and '{' not in output_b): |
| conflicts_detected.append("format conflict: A uses JSON, B uses YAML-style") |
| if (':' in output_b and '"' in output_b) and (':' in output_a and '"' not in output_a and '{' not in output_a): |
| conflicts_detected.append("format conflict: B uses JSON, A uses YAML-style") |
|
|
| verification = verify_against_spec(merged, spec) |
|
|
| merge_succeeded = len(conflicts_detected) == 0 and verification["spec_satisfied"] |
|
|
| return { |
| "merged": merged, |
| "merge_succeeded": merge_succeeded, |
| "conflicts_detected": conflicts_detected, |
| "spec_satisfied": verification["spec_satisfied"], |
| "satisfaction_score": verification["satisfaction_score"], |
| "missing_fields": verification["missing_fields"], |
| "type_mismatches": verification["type_mismatches"], |
| } |
|
|
|
|
| def score_agent_alignment(agent_output: str, spec: dict) -> dict: |
| """ |
| Scores a single agent's output against spec ground truth. |
| Returns 0.0–1.0. Used to compute contrast between agents. |
| """ |
| ground_truth = spec["ground_truth"] |
| score = 0.0 |
| total = 0 |
| issues = [] |
|
|
| expected_fields = _extract_expected_fields(ground_truth) |
| for field in expected_fields: |
| total += 1 |
| if field.lower() in agent_output.lower(): |
| score += 1.0 |
| else: |
| issues.append(f"missing field: {field}") |
|
|
| for identifier_key in ["endpoint", "table", "type_name", "event_name", "command", "error_code"]: |
| if identifier_key in ground_truth: |
| total += 1 |
| if str(ground_truth[identifier_key]).lower() in agent_output.lower(): |
| score += 1.0 |
| else: |
| issues.append(f"wrong {identifier_key}: expected {ground_truth[identifier_key]}") |
|
|
| alignment = score / total if total > 0 else 0.0 |
| return { |
| "alignment_score": round(alignment, 3), |
| "issues": issues, |
| "fields_checked": total, |
| } |
|
|