Add validation-driven workflow decisions
Browse files- backend/app/agents/decision.py +72 -0
- backend/app/agents/validation.py +337 -0
- backend/app/repositories/rule_repository.py +23 -0
- backend/app/workflow/graph.py +128 -25
- backend/app/workflow/nodes/__init__.py +2 -0
- backend/app/workflow/nodes/decision.py +20 -0
- backend/app/workflow/nodes/validate.py +99 -0
- backend/app/workflow/router.py +21 -0
- backend/app/workflow/state.py +5 -0
- backend/test_decision.py +82 -0
- backend/test_router.py +53 -0
- backend/test_validation.py +168 -0
backend/app/agents/decision.py
CHANGED
|
@@ -0,0 +1,72 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
from typing import Any
|
| 4 |
+
|
| 5 |
+
|
| 6 |
+
class DecisionAgent:
|
| 7 |
+
"""
|
| 8 |
+
Makes a deterministic workflow decision from validation results.
|
| 9 |
+
|
| 10 |
+
The decision controls which path the LangGraph workflow takes.
|
| 11 |
+
"""
|
| 12 |
+
|
| 13 |
+
def decide(
|
| 14 |
+
self,
|
| 15 |
+
validation_results: list[dict[str, Any]],
|
| 16 |
+
) -> dict[str, Any]:
|
| 17 |
+
|
| 18 |
+
failures = [
|
| 19 |
+
result
|
| 20 |
+
for result in validation_results
|
| 21 |
+
if result.get("status") == "FAIL"
|
| 22 |
+
]
|
| 23 |
+
|
| 24 |
+
warnings = [
|
| 25 |
+
result
|
| 26 |
+
for result in validation_results
|
| 27 |
+
if result.get("status") == "WARNING"
|
| 28 |
+
]
|
| 29 |
+
|
| 30 |
+
affected_proposal_ids = sorted(
|
| 31 |
+
{
|
| 32 |
+
proposal_id
|
| 33 |
+
for result in validation_results
|
| 34 |
+
if result.get("status") in {"FAIL", "WARNING"}
|
| 35 |
+
for proposal_id in result.get("proposal_ids", [])
|
| 36 |
+
}
|
| 37 |
+
)
|
| 38 |
+
|
| 39 |
+
if failures:
|
| 40 |
+
return {
|
| 41 |
+
"decision": "REVIEW",
|
| 42 |
+
"reason": (
|
| 43 |
+
f"{len(failures)} validation finding(s) "
|
| 44 |
+
"require human review."
|
| 45 |
+
),
|
| 46 |
+
"severity": "HIGH",
|
| 47 |
+
"failure_count": len(failures),
|
| 48 |
+
"warning_count": len(warnings),
|
| 49 |
+
"affected_proposal_ids": affected_proposal_ids,
|
| 50 |
+
}
|
| 51 |
+
|
| 52 |
+
if warnings:
|
| 53 |
+
return {
|
| 54 |
+
"decision": "REVIEW",
|
| 55 |
+
"reason": (
|
| 56 |
+
f"{len(warnings)} validation warning(s) "
|
| 57 |
+
"require human review."
|
| 58 |
+
),
|
| 59 |
+
"severity": "MEDIUM",
|
| 60 |
+
"failure_count": 0,
|
| 61 |
+
"warning_count": len(warnings),
|
| 62 |
+
"affected_proposal_ids": affected_proposal_ids,
|
| 63 |
+
}
|
| 64 |
+
|
| 65 |
+
return {
|
| 66 |
+
"decision": "CONTINUE",
|
| 67 |
+
"reason": "All validation rules passed.",
|
| 68 |
+
"severity": "INFO",
|
| 69 |
+
"failure_count": 0,
|
| 70 |
+
"warning_count": 0,
|
| 71 |
+
"affected_proposal_ids": [],
|
| 72 |
+
}
|
backend/app/agents/validation.py
CHANGED
|
@@ -0,0 +1,337 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
from typing import Any
|
| 4 |
+
|
| 5 |
+
from app.models.proposal import Proposal, ProposalType
|
| 6 |
+
from app.models.rule import Rule, RuleType
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
class RuleValidationAgent:
|
| 10 |
+
"""
|
| 11 |
+
Deterministic rule validation engine.
|
| 12 |
+
|
| 13 |
+
Supported operators:
|
| 14 |
+
|
| 15 |
+
required_evidence
|
| 16 |
+
min_confidence
|
| 17 |
+
allowed_proposal_types
|
| 18 |
+
|
| 19 |
+
Unknown operators produce a WARNING rather than silently passing.
|
| 20 |
+
"""
|
| 21 |
+
|
| 22 |
+
def validate(
|
| 23 |
+
self,
|
| 24 |
+
*,
|
| 25 |
+
rules: list[Rule],
|
| 26 |
+
proposals: list[Proposal],
|
| 27 |
+
) -> list[dict[str, Any]]:
|
| 28 |
+
results: list[dict[str, Any]] = []
|
| 29 |
+
|
| 30 |
+
if not rules:
|
| 31 |
+
return [
|
| 32 |
+
{
|
| 33 |
+
"status": "PASS",
|
| 34 |
+
"severity": "INFO",
|
| 35 |
+
"rule_id": None,
|
| 36 |
+
"rule_name": "No enabled rules",
|
| 37 |
+
"rule_type": None,
|
| 38 |
+
"operator": None,
|
| 39 |
+
"message": "No enabled validation rules are configured.",
|
| 40 |
+
"proposal_ids": [
|
| 41 |
+
str(proposal.id) for proposal in proposals
|
| 42 |
+
],
|
| 43 |
+
}
|
| 44 |
+
]
|
| 45 |
+
|
| 46 |
+
for rule in rules:
|
| 47 |
+
configuration = rule.configuration or {}
|
| 48 |
+
operator = configuration.get("operator")
|
| 49 |
+
|
| 50 |
+
if operator == "required_evidence":
|
| 51 |
+
results.extend(
|
| 52 |
+
self._validate_required_evidence(
|
| 53 |
+
rule=rule,
|
| 54 |
+
proposals=proposals,
|
| 55 |
+
)
|
| 56 |
+
)
|
| 57 |
+
|
| 58 |
+
elif operator == "min_confidence":
|
| 59 |
+
results.extend(
|
| 60 |
+
self._validate_min_confidence(
|
| 61 |
+
rule=rule,
|
| 62 |
+
proposals=proposals,
|
| 63 |
+
)
|
| 64 |
+
)
|
| 65 |
+
|
| 66 |
+
elif operator == "allowed_proposal_types":
|
| 67 |
+
results.extend(
|
| 68 |
+
self._validate_allowed_proposal_types(
|
| 69 |
+
rule=rule,
|
| 70 |
+
proposals=proposals,
|
| 71 |
+
)
|
| 72 |
+
)
|
| 73 |
+
|
| 74 |
+
else:
|
| 75 |
+
results.append(
|
| 76 |
+
{
|
| 77 |
+
"status": "WARNING",
|
| 78 |
+
"severity": "MEDIUM",
|
| 79 |
+
"rule_id": str(rule.id),
|
| 80 |
+
"rule_name": rule.name,
|
| 81 |
+
"rule_type": (
|
| 82 |
+
rule.rule_type.value
|
| 83 |
+
if isinstance(rule.rule_type, RuleType)
|
| 84 |
+
else str(rule.rule_type)
|
| 85 |
+
),
|
| 86 |
+
"operator": operator,
|
| 87 |
+
"message": (
|
| 88 |
+
f"Unsupported rule operator: {operator!r}. "
|
| 89 |
+
"The rule was not evaluated."
|
| 90 |
+
),
|
| 91 |
+
"proposal_ids": [
|
| 92 |
+
str(proposal.id)
|
| 93 |
+
for proposal in proposals
|
| 94 |
+
],
|
| 95 |
+
}
|
| 96 |
+
)
|
| 97 |
+
|
| 98 |
+
return results
|
| 99 |
+
|
| 100 |
+
def _validate_required_evidence(
|
| 101 |
+
self,
|
| 102 |
+
*,
|
| 103 |
+
rule: Rule,
|
| 104 |
+
proposals: list[Proposal],
|
| 105 |
+
) -> list[dict[str, Any]]:
|
| 106 |
+
failures = []
|
| 107 |
+
|
| 108 |
+
for proposal in proposals:
|
| 109 |
+
changes = proposal.proposed_changes or {}
|
| 110 |
+
proposed = changes.get("proposed", changes)
|
| 111 |
+
evidence = proposed.get("evidence")
|
| 112 |
+
|
| 113 |
+
if not evidence:
|
| 114 |
+
failures.append(
|
| 115 |
+
{
|
| 116 |
+
"status": "FAIL",
|
| 117 |
+
"severity": "HIGH",
|
| 118 |
+
"rule_id": str(rule.id),
|
| 119 |
+
"rule_name": rule.name,
|
| 120 |
+
"rule_type": (
|
| 121 |
+
rule.rule_type.value
|
| 122 |
+
if isinstance(rule.rule_type, RuleType)
|
| 123 |
+
else str(rule.rule_type)
|
| 124 |
+
),
|
| 125 |
+
"operator": "required_evidence",
|
| 126 |
+
"message": (
|
| 127 |
+
"Proposal does not contain evidence metadata "
|
| 128 |
+
"in proposed_changes."
|
| 129 |
+
),
|
| 130 |
+
"proposal_ids": [str(proposal.id)],
|
| 131 |
+
}
|
| 132 |
+
)
|
| 133 |
+
|
| 134 |
+
if failures:
|
| 135 |
+
return failures
|
| 136 |
+
|
| 137 |
+
return [
|
| 138 |
+
{
|
| 139 |
+
"status": "PASS",
|
| 140 |
+
"severity": "INFO",
|
| 141 |
+
"rule_id": str(rule.id),
|
| 142 |
+
"rule_name": rule.name,
|
| 143 |
+
"rule_type": (
|
| 144 |
+
rule.rule_type.value
|
| 145 |
+
if isinstance(rule.rule_type, RuleType)
|
| 146 |
+
else str(rule.rule_type)
|
| 147 |
+
),
|
| 148 |
+
"operator": "required_evidence",
|
| 149 |
+
"message": "All proposals contain evidence metadata.",
|
| 150 |
+
"proposal_ids": [
|
| 151 |
+
str(proposal.id) for proposal in proposals
|
| 152 |
+
],
|
| 153 |
+
}
|
| 154 |
+
]
|
| 155 |
+
|
| 156 |
+
def _validate_min_confidence(
|
| 157 |
+
self,
|
| 158 |
+
*,
|
| 159 |
+
rule: Rule,
|
| 160 |
+
proposals: list[Proposal],
|
| 161 |
+
) -> list[dict[str, Any]]:
|
| 162 |
+
configuration = rule.configuration or {}
|
| 163 |
+
minimum = configuration.get("value")
|
| 164 |
+
|
| 165 |
+
if not isinstance(minimum, (int, float)):
|
| 166 |
+
return [
|
| 167 |
+
{
|
| 168 |
+
"status": "WARNING",
|
| 169 |
+
"severity": "HIGH",
|
| 170 |
+
"rule_id": str(rule.id),
|
| 171 |
+
"rule_name": rule.name,
|
| 172 |
+
"rule_type": (
|
| 173 |
+
rule.rule_type.value
|
| 174 |
+
if isinstance(rule.rule_type, RuleType)
|
| 175 |
+
else str(rule.rule_type)
|
| 176 |
+
),
|
| 177 |
+
"operator": "min_confidence",
|
| 178 |
+
"message": (
|
| 179 |
+
"min_confidence requires a numeric "
|
| 180 |
+
"'value' configuration."
|
| 181 |
+
),
|
| 182 |
+
"proposal_ids": [],
|
| 183 |
+
}
|
| 184 |
+
]
|
| 185 |
+
|
| 186 |
+
results = []
|
| 187 |
+
|
| 188 |
+
for proposal in proposals:
|
| 189 |
+
changes = proposal.proposed_changes or {}
|
| 190 |
+
proposed = changes.get("proposed", changes)
|
| 191 |
+
confidence = proposed.get("confidence")
|
| 192 |
+
|
| 193 |
+
if confidence is None:
|
| 194 |
+
results.append(
|
| 195 |
+
{
|
| 196 |
+
"status": "FAIL",
|
| 197 |
+
"severity": "HIGH",
|
| 198 |
+
"rule_id": str(rule.id),
|
| 199 |
+
"rule_name": rule.name,
|
| 200 |
+
"rule_type": (
|
| 201 |
+
rule.rule_type.value
|
| 202 |
+
if isinstance(rule.rule_type, RuleType)
|
| 203 |
+
else str(rule.rule_type)
|
| 204 |
+
),
|
| 205 |
+
"operator": "min_confidence",
|
| 206 |
+
"message": (
|
| 207 |
+
"Proposal does not contain a confidence value."
|
| 208 |
+
),
|
| 209 |
+
"proposal_ids": [str(proposal.id)],
|
| 210 |
+
}
|
| 211 |
+
)
|
| 212 |
+
continue
|
| 213 |
+
|
| 214 |
+
if confidence < minimum:
|
| 215 |
+
results.append(
|
| 216 |
+
{
|
| 217 |
+
"status": "FAIL",
|
| 218 |
+
"severity": "HIGH",
|
| 219 |
+
"rule_id": str(rule.id),
|
| 220 |
+
"rule_name": rule.name,
|
| 221 |
+
"rule_type": (
|
| 222 |
+
rule.rule_type.value
|
| 223 |
+
if isinstance(rule.rule_type, RuleType)
|
| 224 |
+
else str(rule.rule_type)
|
| 225 |
+
),
|
| 226 |
+
"operator": "min_confidence",
|
| 227 |
+
"message": (
|
| 228 |
+
f"Confidence {confidence:.3f} is below "
|
| 229 |
+
f"required minimum {minimum:.3f}."
|
| 230 |
+
),
|
| 231 |
+
"proposal_ids": [str(proposal.id)],
|
| 232 |
+
}
|
| 233 |
+
)
|
| 234 |
+
else:
|
| 235 |
+
results.append(
|
| 236 |
+
{
|
| 237 |
+
"status": "PASS",
|
| 238 |
+
"severity": "INFO",
|
| 239 |
+
"rule_id": str(rule.id),
|
| 240 |
+
"rule_name": rule.name,
|
| 241 |
+
"rule_type": (
|
| 242 |
+
rule.rule_type.value
|
| 243 |
+
if isinstance(rule.rule_type, RuleType)
|
| 244 |
+
else str(rule.rule_type)
|
| 245 |
+
),
|
| 246 |
+
"operator": "min_confidence",
|
| 247 |
+
"message": (
|
| 248 |
+
f"Confidence {confidence:.3f} meets "
|
| 249 |
+
f"minimum {minimum:.3f}."
|
| 250 |
+
),
|
| 251 |
+
"proposal_ids": [str(proposal.id)],
|
| 252 |
+
}
|
| 253 |
+
)
|
| 254 |
+
|
| 255 |
+
return results
|
| 256 |
+
|
| 257 |
+
def _validate_allowed_proposal_types(
|
| 258 |
+
self,
|
| 259 |
+
*,
|
| 260 |
+
rule: Rule,
|
| 261 |
+
proposals: list[Proposal],
|
| 262 |
+
) -> list[dict[str, Any]]:
|
| 263 |
+
configuration = rule.configuration or {}
|
| 264 |
+
allowed = configuration.get("values")
|
| 265 |
+
|
| 266 |
+
if not isinstance(allowed, list):
|
| 267 |
+
return [
|
| 268 |
+
{
|
| 269 |
+
"status": "WARNING",
|
| 270 |
+
"severity": "HIGH",
|
| 271 |
+
"rule_id": str(rule.id),
|
| 272 |
+
"rule_name": rule.name,
|
| 273 |
+
"rule_type": (
|
| 274 |
+
rule.rule_type.value
|
| 275 |
+
if isinstance(rule.rule_type, RuleType)
|
| 276 |
+
else str(rule.rule_type)
|
| 277 |
+
),
|
| 278 |
+
"operator": "allowed_proposal_types",
|
| 279 |
+
"message": (
|
| 280 |
+
"allowed_proposal_types requires a list "
|
| 281 |
+
"under 'values'."
|
| 282 |
+
),
|
| 283 |
+
"proposal_ids": [],
|
| 284 |
+
}
|
| 285 |
+
]
|
| 286 |
+
|
| 287 |
+
allowed = {str(value).upper() for value in allowed}
|
| 288 |
+
results = []
|
| 289 |
+
|
| 290 |
+
for proposal in proposals:
|
| 291 |
+
proposal_type = (
|
| 292 |
+
proposal.proposal_type.value
|
| 293 |
+
if isinstance(proposal.proposal_type, ProposalType)
|
| 294 |
+
else str(proposal.proposal_type)
|
| 295 |
+
)
|
| 296 |
+
|
| 297 |
+
if proposal_type.upper() not in allowed:
|
| 298 |
+
results.append(
|
| 299 |
+
{
|
| 300 |
+
"status": "FAIL",
|
| 301 |
+
"severity": "HIGH",
|
| 302 |
+
"rule_id": str(rule.id),
|
| 303 |
+
"rule_name": rule.name,
|
| 304 |
+
"rule_type": (
|
| 305 |
+
rule.rule_type.value
|
| 306 |
+
if isinstance(rule.rule_type, RuleType)
|
| 307 |
+
else str(rule.rule_type)
|
| 308 |
+
),
|
| 309 |
+
"operator": "allowed_proposal_types",
|
| 310 |
+
"message": (
|
| 311 |
+
f"Proposal type {proposal_type!r} is not "
|
| 312 |
+
f"allowed by this rule."
|
| 313 |
+
),
|
| 314 |
+
"proposal_ids": [str(proposal.id)],
|
| 315 |
+
}
|
| 316 |
+
)
|
| 317 |
+
else:
|
| 318 |
+
results.append(
|
| 319 |
+
{
|
| 320 |
+
"status": "PASS",
|
| 321 |
+
"severity": "INFO",
|
| 322 |
+
"rule_id": str(rule.id),
|
| 323 |
+
"rule_name": rule.name,
|
| 324 |
+
"rule_type": (
|
| 325 |
+
rule.rule_type.value
|
| 326 |
+
if isinstance(rule.rule_type, RuleType)
|
| 327 |
+
else str(rule.rule_type)
|
| 328 |
+
),
|
| 329 |
+
"operator": "allowed_proposal_types",
|
| 330 |
+
"message": (
|
| 331 |
+
f"Proposal type {proposal_type!r} is allowed."
|
| 332 |
+
),
|
| 333 |
+
"proposal_ids": [str(proposal.id)],
|
| 334 |
+
}
|
| 335 |
+
)
|
| 336 |
+
|
| 337 |
+
return results
|
backend/app/repositories/rule_repository.py
CHANGED
|
@@ -0,0 +1,23 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
from sqlalchemy.orm import Session
|
| 4 |
+
|
| 5 |
+
from app.models.rule import Rule
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
class RuleRepository:
|
| 9 |
+
"""Repository for workspace-scoped validation rules."""
|
| 10 |
+
|
| 11 |
+
def __init__(self, db: Session):
|
| 12 |
+
self.db = db
|
| 13 |
+
|
| 14 |
+
def list_enabled(self, workspace_id) -> list[Rule]:
|
| 15 |
+
return (
|
| 16 |
+
self.db.query(Rule)
|
| 17 |
+
.filter(
|
| 18 |
+
Rule.workspace_id == workspace_id,
|
| 19 |
+
Rule.enabled.is_(True),
|
| 20 |
+
)
|
| 21 |
+
.order_by(Rule.created_at.asc())
|
| 22 |
+
.all()
|
| 23 |
+
)
|
backend/app/workflow/graph.py
CHANGED
|
@@ -1,4 +1,4 @@
|
|
| 1 |
-
from __future__ import annotations
|
| 2 |
|
| 3 |
from sqlalchemy.orm import Session
|
| 4 |
from langgraph.graph import END, START, StateGraph
|
|
@@ -7,6 +7,7 @@ from app.llm.client import LLMClient
|
|
| 7 |
from app.repositories.knowledge_repository import KnowledgeRepository
|
| 8 |
from app.services.chunking_service import ChunkingService
|
| 9 |
from app.services.knowledge_link_service import KnowledgeLinkService
|
|
|
|
| 10 |
|
| 11 |
from app.workflow.nodes.chunk import chunk
|
| 12 |
from app.workflow.nodes.classify import classify
|
|
@@ -15,11 +16,19 @@ from app.workflow.nodes.extract import extract
|
|
| 15 |
from app.workflow.nodes.knowledge import knowledge
|
| 16 |
from app.workflow.nodes.link import link
|
| 17 |
from app.workflow.nodes.reconcile import reconcile
|
| 18 |
-
from app.workflow.state import WorkflowState
|
| 19 |
-
from app.services.embedding_service import EmbeddingService
|
| 20 |
from app.workflow.nodes.embed import embed
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 21 |
|
| 22 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 23 |
"""
|
| 24 |
Build the DocWeave document intelligence workflow.
|
| 25 |
"""
|
|
@@ -29,13 +38,17 @@ def build_workflow(db: Session, checkpointer=None,interrupt_before=None,):
|
|
| 29 |
chunking_service = ChunkingService(db)
|
| 30 |
knowledge_link_service = KnowledgeLinkService(db)
|
| 31 |
embedding_service = EmbeddingService()
|
|
|
|
| 32 |
graph = StateGraph(WorkflowState)
|
| 33 |
|
| 34 |
# ------------------------------------------------------------------
|
| 35 |
# Nodes
|
| 36 |
# ------------------------------------------------------------------
|
| 37 |
|
| 38 |
-
graph.add_node(
|
|
|
|
|
|
|
|
|
|
| 39 |
|
| 40 |
graph.add_node(
|
| 41 |
"chunk",
|
|
@@ -45,7 +58,19 @@ def build_workflow(db: Session, checkpointer=None,interrupt_before=None,):
|
|
| 45 |
),
|
| 46 |
)
|
| 47 |
|
| 48 |
-
graph.add_node(
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 49 |
|
| 50 |
graph.add_node(
|
| 51 |
"knowledge",
|
|
@@ -64,6 +89,19 @@ def build_workflow(db: Session, checkpointer=None,interrupt_before=None,):
|
|
| 64 |
),
|
| 65 |
)
|
| 66 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 67 |
graph.add_node(
|
| 68 |
"link",
|
| 69 |
lambda state: link(
|
|
@@ -72,29 +110,94 @@ def build_workflow(db: Session, checkpointer=None,interrupt_before=None,):
|
|
| 72 |
),
|
| 73 |
)
|
| 74 |
|
| 75 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 76 |
|
| 77 |
graph.add_node(
|
| 78 |
-
"
|
| 79 |
-
|
| 80 |
-
state,
|
| 81 |
-
db,
|
| 82 |
-
embedding_service,
|
| 83 |
-
),
|
| 84 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 85 |
# ------------------------------------------------------------------
|
| 86 |
# Workflow edges
|
| 87 |
# ------------------------------------------------------------------
|
| 88 |
|
| 89 |
-
graph.add_edge(
|
| 90 |
-
|
| 91 |
-
|
| 92 |
-
|
| 93 |
-
|
| 94 |
-
graph.add_edge(
|
| 95 |
-
|
| 96 |
-
|
| 97 |
-
|
| 98 |
-
|
| 99 |
-
|
| 100 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
|
| 3 |
from sqlalchemy.orm import Session
|
| 4 |
from langgraph.graph import END, START, StateGraph
|
|
|
|
| 7 |
from app.repositories.knowledge_repository import KnowledgeRepository
|
| 8 |
from app.services.chunking_service import ChunkingService
|
| 9 |
from app.services.knowledge_link_service import KnowledgeLinkService
|
| 10 |
+
from app.services.embedding_service import EmbeddingService
|
| 11 |
|
| 12 |
from app.workflow.nodes.chunk import chunk
|
| 13 |
from app.workflow.nodes.classify import classify
|
|
|
|
| 16 |
from app.workflow.nodes.knowledge import knowledge
|
| 17 |
from app.workflow.nodes.link import link
|
| 18 |
from app.workflow.nodes.reconcile import reconcile
|
|
|
|
|
|
|
| 19 |
from app.workflow.nodes.embed import embed
|
| 20 |
+
from app.workflow.nodes.validate import validate
|
| 21 |
+
from app.workflow.nodes.decision import decide
|
| 22 |
+
|
| 23 |
+
from app.workflow.router import route_after_decision
|
| 24 |
+
from app.workflow.state import WorkflowState
|
| 25 |
|
| 26 |
+
|
| 27 |
+
def build_workflow(
|
| 28 |
+
db: Session,
|
| 29 |
+
checkpointer=None,
|
| 30 |
+
interrupt_before=None,
|
| 31 |
+
):
|
| 32 |
"""
|
| 33 |
Build the DocWeave document intelligence workflow.
|
| 34 |
"""
|
|
|
|
| 38 |
chunking_service = ChunkingService(db)
|
| 39 |
knowledge_link_service = KnowledgeLinkService(db)
|
| 40 |
embedding_service = EmbeddingService()
|
| 41 |
+
|
| 42 |
graph = StateGraph(WorkflowState)
|
| 43 |
|
| 44 |
# ------------------------------------------------------------------
|
| 45 |
# Nodes
|
| 46 |
# ------------------------------------------------------------------
|
| 47 |
|
| 48 |
+
graph.add_node(
|
| 49 |
+
"extract",
|
| 50 |
+
extract,
|
| 51 |
+
)
|
| 52 |
|
| 53 |
graph.add_node(
|
| 54 |
"chunk",
|
|
|
|
| 58 |
),
|
| 59 |
)
|
| 60 |
|
| 61 |
+
graph.add_node(
|
| 62 |
+
"embedding",
|
| 63 |
+
lambda state: embed(
|
| 64 |
+
state,
|
| 65 |
+
db,
|
| 66 |
+
embedding_service,
|
| 67 |
+
),
|
| 68 |
+
)
|
| 69 |
+
|
| 70 |
+
graph.add_node(
|
| 71 |
+
"classify",
|
| 72 |
+
classify,
|
| 73 |
+
)
|
| 74 |
|
| 75 |
graph.add_node(
|
| 76 |
"knowledge",
|
|
|
|
| 89 |
),
|
| 90 |
)
|
| 91 |
|
| 92 |
+
graph.add_node(
|
| 93 |
+
"validation",
|
| 94 |
+
lambda state: validate(
|
| 95 |
+
state,
|
| 96 |
+
db,
|
| 97 |
+
),
|
| 98 |
+
)
|
| 99 |
+
|
| 100 |
+
graph.add_node(
|
| 101 |
+
"decision",
|
| 102 |
+
decide,
|
| 103 |
+
)
|
| 104 |
+
|
| 105 |
graph.add_node(
|
| 106 |
"link",
|
| 107 |
lambda state: link(
|
|
|
|
| 110 |
),
|
| 111 |
)
|
| 112 |
|
| 113 |
+
# Temporary branch target.
|
| 114 |
+
# This will become the real durable human gate later.
|
| 115 |
+
def human_review(state: WorkflowState) -> WorkflowState:
|
| 116 |
+
state.current_node = "HUMAN_REVIEW"
|
| 117 |
+
return state
|
| 118 |
|
| 119 |
graph.add_node(
|
| 120 |
+
"human_review",
|
| 121 |
+
human_review,
|
|
|
|
|
|
|
|
|
|
|
|
|
| 122 |
)
|
| 123 |
+
|
| 124 |
+
graph.add_node(
|
| 125 |
+
"complete",
|
| 126 |
+
complete,
|
| 127 |
+
)
|
| 128 |
+
|
| 129 |
# ------------------------------------------------------------------
|
| 130 |
# Workflow edges
|
| 131 |
# ------------------------------------------------------------------
|
| 132 |
|
| 133 |
+
graph.add_edge(
|
| 134 |
+
START,
|
| 135 |
+
"extract",
|
| 136 |
+
)
|
| 137 |
+
|
| 138 |
+
graph.add_edge(
|
| 139 |
+
"extract",
|
| 140 |
+
"chunk",
|
| 141 |
+
)
|
| 142 |
+
|
| 143 |
+
graph.add_edge(
|
| 144 |
+
"chunk",
|
| 145 |
+
"embedding",
|
| 146 |
+
)
|
| 147 |
+
|
| 148 |
+
graph.add_edge(
|
| 149 |
+
"embedding",
|
| 150 |
+
"classify",
|
| 151 |
+
)
|
| 152 |
+
|
| 153 |
+
graph.add_edge(
|
| 154 |
+
"classify",
|
| 155 |
+
"knowledge",
|
| 156 |
+
)
|
| 157 |
+
|
| 158 |
+
graph.add_edge(
|
| 159 |
+
"knowledge",
|
| 160 |
+
"reconciliation",
|
| 161 |
+
)
|
| 162 |
+
|
| 163 |
+
graph.add_edge(
|
| 164 |
+
"reconciliation",
|
| 165 |
+
"validation",
|
| 166 |
+
)
|
| 167 |
+
|
| 168 |
+
graph.add_edge(
|
| 169 |
+
"validation",
|
| 170 |
+
"decision",
|
| 171 |
+
)
|
| 172 |
+
|
| 173 |
+
# Decision controls the workflow path.
|
| 174 |
+
graph.add_conditional_edges(
|
| 175 |
+
"decision",
|
| 176 |
+
route_after_decision,
|
| 177 |
+
{
|
| 178 |
+
"link": "link",
|
| 179 |
+
"human_review": "human_review",
|
| 180 |
+
},
|
| 181 |
+
)
|
| 182 |
+
|
| 183 |
+
graph.add_edge(
|
| 184 |
+
"link",
|
| 185 |
+
"complete",
|
| 186 |
+
)
|
| 187 |
+
|
| 188 |
+
# Temporary review branch returns to completion.
|
| 189 |
+
# We will replace this with the real human gate later.
|
| 190 |
+
graph.add_edge(
|
| 191 |
+
"human_review",
|
| 192 |
+
"complete",
|
| 193 |
+
)
|
| 194 |
+
|
| 195 |
+
graph.add_edge(
|
| 196 |
+
"complete",
|
| 197 |
+
END,
|
| 198 |
+
)
|
| 199 |
+
|
| 200 |
+
return graph.compile(
|
| 201 |
+
checkpointer=checkpointer,
|
| 202 |
+
interrupt_before=interrupt_before,
|
| 203 |
+
)
|
backend/app/workflow/nodes/__init__.py
CHANGED
|
@@ -1,9 +1,11 @@
|
|
| 1 |
from .extract import extract
|
| 2 |
from .classify import classify
|
| 3 |
from .complete import complete
|
|
|
|
| 4 |
|
| 5 |
__all__ = [
|
| 6 |
"extract",
|
| 7 |
"classify",
|
| 8 |
"complete",
|
|
|
|
| 9 |
]
|
|
|
|
| 1 |
from .extract import extract
|
| 2 |
from .classify import classify
|
| 3 |
from .complete import complete
|
| 4 |
+
from .validate import validate
|
| 5 |
|
| 6 |
__all__ = [
|
| 7 |
"extract",
|
| 8 |
"classify",
|
| 9 |
"complete",
|
| 10 |
+
"validate",
|
| 11 |
]
|
backend/app/workflow/nodes/decision.py
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
from app.agents.decision import DecisionAgent
|
| 4 |
+
from app.workflow.state import WorkflowState
|
| 5 |
+
|
| 6 |
+
|
| 7 |
+
def decide(state: WorkflowState) -> WorkflowState:
|
| 8 |
+
"""
|
| 9 |
+
Convert validation findings into a workflow decision.
|
| 10 |
+
"""
|
| 11 |
+
|
| 12 |
+
state.current_node = "DECISION"
|
| 13 |
+
|
| 14 |
+
agent = DecisionAgent()
|
| 15 |
+
|
| 16 |
+
state.decision = agent.decide(
|
| 17 |
+
state.validation_results
|
| 18 |
+
)
|
| 19 |
+
|
| 20 |
+
return state
|
backend/app/workflow/nodes/validate.py
ADDED
|
@@ -0,0 +1,99 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
from app.agents.validation import RuleValidationAgent
|
| 4 |
+
from app.models.proposal import Proposal, ProposalType, ProposalStatus
|
| 5 |
+
from app.repositories.knowledge_repository import KnowledgeRepository
|
| 6 |
+
from app.repositories.rule_repository import RuleRepository
|
| 7 |
+
from app.workflow.state import WorkflowState
|
| 8 |
+
|
| 9 |
+
|
| 10 |
+
def validate(
|
| 11 |
+
state: WorkflowState,
|
| 12 |
+
db,
|
| 13 |
+
) -> WorkflowState:
|
| 14 |
+
"""
|
| 15 |
+
Validate proposals generated by reconciliation against
|
| 16 |
+
enabled workspace rules.
|
| 17 |
+
"""
|
| 18 |
+
|
| 19 |
+
state.current_node = "RULE_VALIDATION"
|
| 20 |
+
|
| 21 |
+
knowledge_repository = KnowledgeRepository(db)
|
| 22 |
+
rule_repository = RuleRepository(db)
|
| 23 |
+
|
| 24 |
+
new_items = knowledge_repository.list_by_document_version(
|
| 25 |
+
state.document_version_id
|
| 26 |
+
)
|
| 27 |
+
|
| 28 |
+
new_item_ids = {str(item.id) for item in new_items}
|
| 29 |
+
|
| 30 |
+
if not new_item_ids:
|
| 31 |
+
state.validation_results = []
|
| 32 |
+
state.metadata["validation_summary"] = {
|
| 33 |
+
"status": "PASS",
|
| 34 |
+
"rules_evaluated": 0,
|
| 35 |
+
"proposals_evaluated": 0,
|
| 36 |
+
"failures": 0,
|
| 37 |
+
"warnings": 0,
|
| 38 |
+
}
|
| 39 |
+
return state
|
| 40 |
+
|
| 41 |
+
proposals = (
|
| 42 |
+
db.query(Proposal)
|
| 43 |
+
.filter(
|
| 44 |
+
Proposal.workspace_id == state.workspace_id,
|
| 45 |
+
Proposal.status == ProposalStatus.PENDING,
|
| 46 |
+
)
|
| 47 |
+
.all()
|
| 48 |
+
)
|
| 49 |
+
|
| 50 |
+
relevant_proposals = []
|
| 51 |
+
|
| 52 |
+
for proposal in proposals:
|
| 53 |
+
changes = proposal.proposed_changes or {}
|
| 54 |
+
|
| 55 |
+
if proposal.proposal_type == ProposalType.CREATE:
|
| 56 |
+
if proposal.knowledge_item_id is not None:
|
| 57 |
+
if str(proposal.knowledge_item_id) in new_item_ids:
|
| 58 |
+
relevant_proposals.append(proposal)
|
| 59 |
+
|
| 60 |
+
elif proposal.proposal_type == ProposalType.UPDATE:
|
| 61 |
+
source_id = changes.get("source_knowledge_item_id")
|
| 62 |
+
|
| 63 |
+
if source_id and str(source_id) in new_item_ids:
|
| 64 |
+
relevant_proposals.append(proposal)
|
| 65 |
+
|
| 66 |
+
rules = rule_repository.list_enabled(state.workspace_id)
|
| 67 |
+
|
| 68 |
+
agent = RuleValidationAgent()
|
| 69 |
+
|
| 70 |
+
results = agent.validate(
|
| 71 |
+
rules=rules,
|
| 72 |
+
proposals=relevant_proposals,
|
| 73 |
+
)
|
| 74 |
+
|
| 75 |
+
state.validation_results = results
|
| 76 |
+
|
| 77 |
+
failures = sum(
|
| 78 |
+
1 for result in results if result["status"] == "FAIL"
|
| 79 |
+
)
|
| 80 |
+
|
| 81 |
+
warnings = sum(
|
| 82 |
+
1 for result in results if result["status"] == "WARNING"
|
| 83 |
+
)
|
| 84 |
+
|
| 85 |
+
state.metadata["validation_summary"] = {
|
| 86 |
+
"status": (
|
| 87 |
+
"FAIL"
|
| 88 |
+
if failures
|
| 89 |
+
else "WARNING"
|
| 90 |
+
if warnings
|
| 91 |
+
else "PASS"
|
| 92 |
+
),
|
| 93 |
+
"rules_evaluated": len(rules),
|
| 94 |
+
"proposals_evaluated": len(relevant_proposals),
|
| 95 |
+
"failures": failures,
|
| 96 |
+
"warnings": warnings,
|
| 97 |
+
}
|
| 98 |
+
|
| 99 |
+
return state
|
backend/app/workflow/router.py
CHANGED
|
@@ -0,0 +1,21 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
from typing import Any
|
| 4 |
+
|
| 5 |
+
|
| 6 |
+
def route_after_decision(state: Any) -> str:
|
| 7 |
+
"""
|
| 8 |
+
Route the workflow based on the decision stored in WorkflowState.
|
| 9 |
+
"""
|
| 10 |
+
|
| 11 |
+
decision = state.decision.get("decision")
|
| 12 |
+
|
| 13 |
+
if decision == "CONTINUE":
|
| 14 |
+
return "link"
|
| 15 |
+
|
| 16 |
+
if decision == "REVIEW":
|
| 17 |
+
return "human_review"
|
| 18 |
+
|
| 19 |
+
raise ValueError(
|
| 20 |
+
f"Unknown workflow decision: {decision!r}"
|
| 21 |
+
)
|
backend/app/workflow/state.py
CHANGED
|
@@ -29,6 +29,11 @@ class WorkflowState:
|
|
| 29 |
|
| 30 |
metadata: dict[str, Any] = field(default_factory=dict)
|
| 31 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 32 |
current_node: str = "START"
|
| 33 |
|
| 34 |
completed: bool = False
|
|
|
|
| 29 |
|
| 30 |
metadata: dict[str, Any] = field(default_factory=dict)
|
| 31 |
|
| 32 |
+
validation_results: list[dict[str, Any]] = field(
|
| 33 |
+
default_factory=list
|
| 34 |
+
)
|
| 35 |
+
|
| 36 |
+
decision: dict[str, Any] = field(default_factory=dict)
|
| 37 |
current_node: str = "START"
|
| 38 |
|
| 39 |
completed: bool = False
|
backend/test_decision.py
ADDED
|
@@ -0,0 +1,82 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from app.agents.decision import DecisionAgent
|
| 2 |
+
|
| 3 |
+
|
| 4 |
+
agent = DecisionAgent()
|
| 5 |
+
|
| 6 |
+
|
| 7 |
+
print("\nTEST 1 - ALL PASS")
|
| 8 |
+
|
| 9 |
+
result = agent.decide([
|
| 10 |
+
{
|
| 11 |
+
"status": "PASS",
|
| 12 |
+
"severity": "INFO",
|
| 13 |
+
"proposal_ids": ["proposal-1"],
|
| 14 |
+
}
|
| 15 |
+
])
|
| 16 |
+
|
| 17 |
+
print(result)
|
| 18 |
+
|
| 19 |
+
assert result["decision"] == "CONTINUE"
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
print("\nTEST 2 - FAIL")
|
| 23 |
+
|
| 24 |
+
result = agent.decide([
|
| 25 |
+
{
|
| 26 |
+
"status": "FAIL",
|
| 27 |
+
"severity": "HIGH",
|
| 28 |
+
"proposal_ids": ["proposal-2"],
|
| 29 |
+
}
|
| 30 |
+
])
|
| 31 |
+
|
| 32 |
+
print(result)
|
| 33 |
+
|
| 34 |
+
assert result["decision"] == "REVIEW"
|
| 35 |
+
assert result["failure_count"] == 1
|
| 36 |
+
assert "proposal-2" in result["affected_proposal_ids"]
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
print("\nTEST 3 - WARNING")
|
| 40 |
+
|
| 41 |
+
result = agent.decide([
|
| 42 |
+
{
|
| 43 |
+
"status": "WARNING",
|
| 44 |
+
"severity": "MEDIUM",
|
| 45 |
+
"proposal_ids": ["proposal-3"],
|
| 46 |
+
}
|
| 47 |
+
])
|
| 48 |
+
|
| 49 |
+
print(result)
|
| 50 |
+
|
| 51 |
+
assert result["decision"] == "REVIEW"
|
| 52 |
+
assert result["warning_count"] == 1
|
| 53 |
+
assert "proposal-3" in result["affected_proposal_ids"]
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
print("\nTEST 4 - FAIL + WARNING")
|
| 57 |
+
|
| 58 |
+
result = agent.decide([
|
| 59 |
+
{
|
| 60 |
+
"status": "FAIL",
|
| 61 |
+
"severity": "HIGH",
|
| 62 |
+
"proposal_ids": ["proposal-4"],
|
| 63 |
+
},
|
| 64 |
+
{
|
| 65 |
+
"status": "WARNING",
|
| 66 |
+
"severity": "MEDIUM",
|
| 67 |
+
"proposal_ids": ["proposal-5"],
|
| 68 |
+
}
|
| 69 |
+
])
|
| 70 |
+
|
| 71 |
+
print(result)
|
| 72 |
+
|
| 73 |
+
assert result["decision"] == "REVIEW"
|
| 74 |
+
assert result["failure_count"] == 1
|
| 75 |
+
assert result["warning_count"] == 1
|
| 76 |
+
assert "proposal-4" in result["affected_proposal_ids"]
|
| 77 |
+
assert "proposal-5" in result["affected_proposal_ids"]
|
| 78 |
+
|
| 79 |
+
|
| 80 |
+
print("\n===================================")
|
| 81 |
+
print("ALL DECISION TESTS PASSED")
|
| 82 |
+
print("===================================")
|
backend/test_router.py
ADDED
|
@@ -0,0 +1,53 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from types import SimpleNamespace
|
| 2 |
+
|
| 3 |
+
from app.workflow.router import route_after_decision
|
| 4 |
+
|
| 5 |
+
|
| 6 |
+
print("\nTEST 1 - CONTINUE")
|
| 7 |
+
|
| 8 |
+
state = SimpleNamespace(
|
| 9 |
+
decision={
|
| 10 |
+
"decision": "CONTINUE",
|
| 11 |
+
}
|
| 12 |
+
)
|
| 13 |
+
|
| 14 |
+
result = route_after_decision(state)
|
| 15 |
+
|
| 16 |
+
print("ROUTE:", result)
|
| 17 |
+
|
| 18 |
+
assert result == "link"
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
print("\nTEST 2 - REVIEW")
|
| 22 |
+
|
| 23 |
+
state = SimpleNamespace(
|
| 24 |
+
decision={
|
| 25 |
+
"decision": "REVIEW",
|
| 26 |
+
}
|
| 27 |
+
)
|
| 28 |
+
|
| 29 |
+
result = route_after_decision(state)
|
| 30 |
+
|
| 31 |
+
print("ROUTE:", result)
|
| 32 |
+
|
| 33 |
+
assert result == "human_review"
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
print("\nTEST 3 - UNKNOWN DECISION")
|
| 37 |
+
|
| 38 |
+
state = SimpleNamespace(
|
| 39 |
+
decision={
|
| 40 |
+
"decision": "SOMETHING_ELSE",
|
| 41 |
+
}
|
| 42 |
+
)
|
| 43 |
+
|
| 44 |
+
try:
|
| 45 |
+
route_after_decision(state)
|
| 46 |
+
raise AssertionError("Expected ValueError")
|
| 47 |
+
except ValueError as exc:
|
| 48 |
+
print("EXPECTED ERROR:", exc)
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
print("\n===================================")
|
| 52 |
+
print("ALL ROUTER TESTS PASSED")
|
| 53 |
+
print("===================================")
|
backend/test_validation.py
ADDED
|
@@ -0,0 +1,168 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from types import SimpleNamespace
|
| 2 |
+
from uuid import uuid4
|
| 3 |
+
|
| 4 |
+
from app.agents.validation import RuleValidationAgent
|
| 5 |
+
|
| 6 |
+
|
| 7 |
+
agent = RuleValidationAgent()
|
| 8 |
+
|
| 9 |
+
|
| 10 |
+
def make_rule(name, configuration):
|
| 11 |
+
return SimpleNamespace(
|
| 12 |
+
id=uuid4(),
|
| 13 |
+
name=name,
|
| 14 |
+
description="test rule",
|
| 15 |
+
rule_type="COMPLIANCE",
|
| 16 |
+
configuration=configuration,
|
| 17 |
+
enabled=True,
|
| 18 |
+
)
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
def make_proposal(confidence=0.90, evidence=True):
|
| 22 |
+
proposed = {
|
| 23 |
+
"confidence": confidence,
|
| 24 |
+
}
|
| 25 |
+
|
| 26 |
+
if evidence:
|
| 27 |
+
proposed["evidence"] = {
|
| 28 |
+
"source": "test.pdf",
|
| 29 |
+
"page": 1,
|
| 30 |
+
}
|
| 31 |
+
|
| 32 |
+
return SimpleNamespace(
|
| 33 |
+
id=uuid4(),
|
| 34 |
+
proposal_type="CREATE",
|
| 35 |
+
proposed_changes=proposed,
|
| 36 |
+
)
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
# TEST 1: No rules
|
| 40 |
+
result = agent.validate(
|
| 41 |
+
rules=[],
|
| 42 |
+
proposals=[make_proposal()],
|
| 43 |
+
)
|
| 44 |
+
|
| 45 |
+
print("\nTEST 1 - NO RULES")
|
| 46 |
+
print(result)
|
| 47 |
+
|
| 48 |
+
|
| 49 |
+
# TEST 2: Confidence passes
|
| 50 |
+
result = agent.validate(
|
| 51 |
+
rules=[
|
| 52 |
+
make_rule(
|
| 53 |
+
"Minimum confidence",
|
| 54 |
+
{
|
| 55 |
+
"operator": "min_confidence",
|
| 56 |
+
"value": 0.80,
|
| 57 |
+
},
|
| 58 |
+
)
|
| 59 |
+
],
|
| 60 |
+
proposals=[
|
| 61 |
+
make_proposal(confidence=0.90),
|
| 62 |
+
],
|
| 63 |
+
)
|
| 64 |
+
|
| 65 |
+
print("\nTEST 2 - CONFIDENCE PASS")
|
| 66 |
+
print(result)
|
| 67 |
+
|
| 68 |
+
|
| 69 |
+
# TEST 3: Confidence fails
|
| 70 |
+
result = agent.validate(
|
| 71 |
+
rules=[
|
| 72 |
+
make_rule(
|
| 73 |
+
"Minimum confidence",
|
| 74 |
+
{
|
| 75 |
+
"operator": "min_confidence",
|
| 76 |
+
"value": 0.80,
|
| 77 |
+
},
|
| 78 |
+
)
|
| 79 |
+
],
|
| 80 |
+
proposals=[
|
| 81 |
+
make_proposal(confidence=0.50),
|
| 82 |
+
],
|
| 83 |
+
)
|
| 84 |
+
|
| 85 |
+
print("\nTEST 3 - CONFIDENCE FAIL")
|
| 86 |
+
print(result)
|
| 87 |
+
|
| 88 |
+
|
| 89 |
+
# TEST 4: Evidence passes
|
| 90 |
+
result = agent.validate(
|
| 91 |
+
rules=[
|
| 92 |
+
make_rule(
|
| 93 |
+
"Required evidence",
|
| 94 |
+
{
|
| 95 |
+
"operator": "required_evidence",
|
| 96 |
+
},
|
| 97 |
+
)
|
| 98 |
+
],
|
| 99 |
+
proposals=[
|
| 100 |
+
make_proposal(evidence=True),
|
| 101 |
+
],
|
| 102 |
+
)
|
| 103 |
+
|
| 104 |
+
print("\nTEST 4 - EVIDENCE PASS")
|
| 105 |
+
print(result)
|
| 106 |
+
|
| 107 |
+
|
| 108 |
+
# TEST 5: Evidence fails
|
| 109 |
+
result = agent.validate(
|
| 110 |
+
rules=[
|
| 111 |
+
make_rule(
|
| 112 |
+
"Required evidence",
|
| 113 |
+
{
|
| 114 |
+
"operator": "required_evidence",
|
| 115 |
+
},
|
| 116 |
+
)
|
| 117 |
+
],
|
| 118 |
+
proposals=[
|
| 119 |
+
make_proposal(evidence=False),
|
| 120 |
+
],
|
| 121 |
+
)
|
| 122 |
+
|
| 123 |
+
print("\nTEST 5 - EVIDENCE FAIL")
|
| 124 |
+
print(result)
|
| 125 |
+
|
| 126 |
+
|
| 127 |
+
# TEST 6: Malformed rule
|
| 128 |
+
result = agent.validate(
|
| 129 |
+
rules=[
|
| 130 |
+
make_rule(
|
| 131 |
+
"Broken confidence rule",
|
| 132 |
+
{
|
| 133 |
+
"operator": "min_confidence",
|
| 134 |
+
"value": "not-a-number",
|
| 135 |
+
},
|
| 136 |
+
)
|
| 137 |
+
],
|
| 138 |
+
proposals=[
|
| 139 |
+
make_proposal(),
|
| 140 |
+
],
|
| 141 |
+
)
|
| 142 |
+
|
| 143 |
+
print("\nTEST 6 - MALFORMED RULE")
|
| 144 |
+
print(result)
|
| 145 |
+
|
| 146 |
+
|
| 147 |
+
# TEST 7: Unsupported operator
|
| 148 |
+
result = agent.validate(
|
| 149 |
+
rules=[
|
| 150 |
+
make_rule(
|
| 151 |
+
"Unknown rule",
|
| 152 |
+
{
|
| 153 |
+
"operator": "unsupported_operator",
|
| 154 |
+
},
|
| 155 |
+
)
|
| 156 |
+
],
|
| 157 |
+
proposals=[
|
| 158 |
+
make_proposal(),
|
| 159 |
+
],
|
| 160 |
+
)
|
| 161 |
+
|
| 162 |
+
print("\nTEST 7 - UNKNOWN OPERATOR")
|
| 163 |
+
print(result)
|
| 164 |
+
|
| 165 |
+
|
| 166 |
+
print("\n===================================")
|
| 167 |
+
print("VALIDATION TEST RUN COMPLETE")
|
| 168 |
+
print("===================================")
|