shak3008 commited on
Commit
d1644cd
·
1 Parent(s): 9308228

Add validation-driven workflow decisions

Browse files
backend/app/agents/decision.py CHANGED
@@ -0,0 +1,72 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ from typing import Any
4
+
5
+
6
+ class DecisionAgent:
7
+ """
8
+ Makes a deterministic workflow decision from validation results.
9
+
10
+ The decision controls which path the LangGraph workflow takes.
11
+ """
12
+
13
+ def decide(
14
+ self,
15
+ validation_results: list[dict[str, Any]],
16
+ ) -> dict[str, Any]:
17
+
18
+ failures = [
19
+ result
20
+ for result in validation_results
21
+ if result.get("status") == "FAIL"
22
+ ]
23
+
24
+ warnings = [
25
+ result
26
+ for result in validation_results
27
+ if result.get("status") == "WARNING"
28
+ ]
29
+
30
+ affected_proposal_ids = sorted(
31
+ {
32
+ proposal_id
33
+ for result in validation_results
34
+ if result.get("status") in {"FAIL", "WARNING"}
35
+ for proposal_id in result.get("proposal_ids", [])
36
+ }
37
+ )
38
+
39
+ if failures:
40
+ return {
41
+ "decision": "REVIEW",
42
+ "reason": (
43
+ f"{len(failures)} validation finding(s) "
44
+ "require human review."
45
+ ),
46
+ "severity": "HIGH",
47
+ "failure_count": len(failures),
48
+ "warning_count": len(warnings),
49
+ "affected_proposal_ids": affected_proposal_ids,
50
+ }
51
+
52
+ if warnings:
53
+ return {
54
+ "decision": "REVIEW",
55
+ "reason": (
56
+ f"{len(warnings)} validation warning(s) "
57
+ "require human review."
58
+ ),
59
+ "severity": "MEDIUM",
60
+ "failure_count": 0,
61
+ "warning_count": len(warnings),
62
+ "affected_proposal_ids": affected_proposal_ids,
63
+ }
64
+
65
+ return {
66
+ "decision": "CONTINUE",
67
+ "reason": "All validation rules passed.",
68
+ "severity": "INFO",
69
+ "failure_count": 0,
70
+ "warning_count": 0,
71
+ "affected_proposal_ids": [],
72
+ }
backend/app/agents/validation.py CHANGED
@@ -0,0 +1,337 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ from typing import Any
4
+
5
+ from app.models.proposal import Proposal, ProposalType
6
+ from app.models.rule import Rule, RuleType
7
+
8
+
9
+ class RuleValidationAgent:
10
+ """
11
+ Deterministic rule validation engine.
12
+
13
+ Supported operators:
14
+
15
+ required_evidence
16
+ min_confidence
17
+ allowed_proposal_types
18
+
19
+ Unknown operators produce a WARNING rather than silently passing.
20
+ """
21
+
22
+ def validate(
23
+ self,
24
+ *,
25
+ rules: list[Rule],
26
+ proposals: list[Proposal],
27
+ ) -> list[dict[str, Any]]:
28
+ results: list[dict[str, Any]] = []
29
+
30
+ if not rules:
31
+ return [
32
+ {
33
+ "status": "PASS",
34
+ "severity": "INFO",
35
+ "rule_id": None,
36
+ "rule_name": "No enabled rules",
37
+ "rule_type": None,
38
+ "operator": None,
39
+ "message": "No enabled validation rules are configured.",
40
+ "proposal_ids": [
41
+ str(proposal.id) for proposal in proposals
42
+ ],
43
+ }
44
+ ]
45
+
46
+ for rule in rules:
47
+ configuration = rule.configuration or {}
48
+ operator = configuration.get("operator")
49
+
50
+ if operator == "required_evidence":
51
+ results.extend(
52
+ self._validate_required_evidence(
53
+ rule=rule,
54
+ proposals=proposals,
55
+ )
56
+ )
57
+
58
+ elif operator == "min_confidence":
59
+ results.extend(
60
+ self._validate_min_confidence(
61
+ rule=rule,
62
+ proposals=proposals,
63
+ )
64
+ )
65
+
66
+ elif operator == "allowed_proposal_types":
67
+ results.extend(
68
+ self._validate_allowed_proposal_types(
69
+ rule=rule,
70
+ proposals=proposals,
71
+ )
72
+ )
73
+
74
+ else:
75
+ results.append(
76
+ {
77
+ "status": "WARNING",
78
+ "severity": "MEDIUM",
79
+ "rule_id": str(rule.id),
80
+ "rule_name": rule.name,
81
+ "rule_type": (
82
+ rule.rule_type.value
83
+ if isinstance(rule.rule_type, RuleType)
84
+ else str(rule.rule_type)
85
+ ),
86
+ "operator": operator,
87
+ "message": (
88
+ f"Unsupported rule operator: {operator!r}. "
89
+ "The rule was not evaluated."
90
+ ),
91
+ "proposal_ids": [
92
+ str(proposal.id)
93
+ for proposal in proposals
94
+ ],
95
+ }
96
+ )
97
+
98
+ return results
99
+
100
+ def _validate_required_evidence(
101
+ self,
102
+ *,
103
+ rule: Rule,
104
+ proposals: list[Proposal],
105
+ ) -> list[dict[str, Any]]:
106
+ failures = []
107
+
108
+ for proposal in proposals:
109
+ changes = proposal.proposed_changes or {}
110
+ proposed = changes.get("proposed", changes)
111
+ evidence = proposed.get("evidence")
112
+
113
+ if not evidence:
114
+ failures.append(
115
+ {
116
+ "status": "FAIL",
117
+ "severity": "HIGH",
118
+ "rule_id": str(rule.id),
119
+ "rule_name": rule.name,
120
+ "rule_type": (
121
+ rule.rule_type.value
122
+ if isinstance(rule.rule_type, RuleType)
123
+ else str(rule.rule_type)
124
+ ),
125
+ "operator": "required_evidence",
126
+ "message": (
127
+ "Proposal does not contain evidence metadata "
128
+ "in proposed_changes."
129
+ ),
130
+ "proposal_ids": [str(proposal.id)],
131
+ }
132
+ )
133
+
134
+ if failures:
135
+ return failures
136
+
137
+ return [
138
+ {
139
+ "status": "PASS",
140
+ "severity": "INFO",
141
+ "rule_id": str(rule.id),
142
+ "rule_name": rule.name,
143
+ "rule_type": (
144
+ rule.rule_type.value
145
+ if isinstance(rule.rule_type, RuleType)
146
+ else str(rule.rule_type)
147
+ ),
148
+ "operator": "required_evidence",
149
+ "message": "All proposals contain evidence metadata.",
150
+ "proposal_ids": [
151
+ str(proposal.id) for proposal in proposals
152
+ ],
153
+ }
154
+ ]
155
+
156
+ def _validate_min_confidence(
157
+ self,
158
+ *,
159
+ rule: Rule,
160
+ proposals: list[Proposal],
161
+ ) -> list[dict[str, Any]]:
162
+ configuration = rule.configuration or {}
163
+ minimum = configuration.get("value")
164
+
165
+ if not isinstance(minimum, (int, float)):
166
+ return [
167
+ {
168
+ "status": "WARNING",
169
+ "severity": "HIGH",
170
+ "rule_id": str(rule.id),
171
+ "rule_name": rule.name,
172
+ "rule_type": (
173
+ rule.rule_type.value
174
+ if isinstance(rule.rule_type, RuleType)
175
+ else str(rule.rule_type)
176
+ ),
177
+ "operator": "min_confidence",
178
+ "message": (
179
+ "min_confidence requires a numeric "
180
+ "'value' configuration."
181
+ ),
182
+ "proposal_ids": [],
183
+ }
184
+ ]
185
+
186
+ results = []
187
+
188
+ for proposal in proposals:
189
+ changes = proposal.proposed_changes or {}
190
+ proposed = changes.get("proposed", changes)
191
+ confidence = proposed.get("confidence")
192
+
193
+ if confidence is None:
194
+ results.append(
195
+ {
196
+ "status": "FAIL",
197
+ "severity": "HIGH",
198
+ "rule_id": str(rule.id),
199
+ "rule_name": rule.name,
200
+ "rule_type": (
201
+ rule.rule_type.value
202
+ if isinstance(rule.rule_type, RuleType)
203
+ else str(rule.rule_type)
204
+ ),
205
+ "operator": "min_confidence",
206
+ "message": (
207
+ "Proposal does not contain a confidence value."
208
+ ),
209
+ "proposal_ids": [str(proposal.id)],
210
+ }
211
+ )
212
+ continue
213
+
214
+ if confidence < minimum:
215
+ results.append(
216
+ {
217
+ "status": "FAIL",
218
+ "severity": "HIGH",
219
+ "rule_id": str(rule.id),
220
+ "rule_name": rule.name,
221
+ "rule_type": (
222
+ rule.rule_type.value
223
+ if isinstance(rule.rule_type, RuleType)
224
+ else str(rule.rule_type)
225
+ ),
226
+ "operator": "min_confidence",
227
+ "message": (
228
+ f"Confidence {confidence:.3f} is below "
229
+ f"required minimum {minimum:.3f}."
230
+ ),
231
+ "proposal_ids": [str(proposal.id)],
232
+ }
233
+ )
234
+ else:
235
+ results.append(
236
+ {
237
+ "status": "PASS",
238
+ "severity": "INFO",
239
+ "rule_id": str(rule.id),
240
+ "rule_name": rule.name,
241
+ "rule_type": (
242
+ rule.rule_type.value
243
+ if isinstance(rule.rule_type, RuleType)
244
+ else str(rule.rule_type)
245
+ ),
246
+ "operator": "min_confidence",
247
+ "message": (
248
+ f"Confidence {confidence:.3f} meets "
249
+ f"minimum {minimum:.3f}."
250
+ ),
251
+ "proposal_ids": [str(proposal.id)],
252
+ }
253
+ )
254
+
255
+ return results
256
+
257
+ def _validate_allowed_proposal_types(
258
+ self,
259
+ *,
260
+ rule: Rule,
261
+ proposals: list[Proposal],
262
+ ) -> list[dict[str, Any]]:
263
+ configuration = rule.configuration or {}
264
+ allowed = configuration.get("values")
265
+
266
+ if not isinstance(allowed, list):
267
+ return [
268
+ {
269
+ "status": "WARNING",
270
+ "severity": "HIGH",
271
+ "rule_id": str(rule.id),
272
+ "rule_name": rule.name,
273
+ "rule_type": (
274
+ rule.rule_type.value
275
+ if isinstance(rule.rule_type, RuleType)
276
+ else str(rule.rule_type)
277
+ ),
278
+ "operator": "allowed_proposal_types",
279
+ "message": (
280
+ "allowed_proposal_types requires a list "
281
+ "under 'values'."
282
+ ),
283
+ "proposal_ids": [],
284
+ }
285
+ ]
286
+
287
+ allowed = {str(value).upper() for value in allowed}
288
+ results = []
289
+
290
+ for proposal in proposals:
291
+ proposal_type = (
292
+ proposal.proposal_type.value
293
+ if isinstance(proposal.proposal_type, ProposalType)
294
+ else str(proposal.proposal_type)
295
+ )
296
+
297
+ if proposal_type.upper() not in allowed:
298
+ results.append(
299
+ {
300
+ "status": "FAIL",
301
+ "severity": "HIGH",
302
+ "rule_id": str(rule.id),
303
+ "rule_name": rule.name,
304
+ "rule_type": (
305
+ rule.rule_type.value
306
+ if isinstance(rule.rule_type, RuleType)
307
+ else str(rule.rule_type)
308
+ ),
309
+ "operator": "allowed_proposal_types",
310
+ "message": (
311
+ f"Proposal type {proposal_type!r} is not "
312
+ f"allowed by this rule."
313
+ ),
314
+ "proposal_ids": [str(proposal.id)],
315
+ }
316
+ )
317
+ else:
318
+ results.append(
319
+ {
320
+ "status": "PASS",
321
+ "severity": "INFO",
322
+ "rule_id": str(rule.id),
323
+ "rule_name": rule.name,
324
+ "rule_type": (
325
+ rule.rule_type.value
326
+ if isinstance(rule.rule_type, RuleType)
327
+ else str(rule.rule_type)
328
+ ),
329
+ "operator": "allowed_proposal_types",
330
+ "message": (
331
+ f"Proposal type {proposal_type!r} is allowed."
332
+ ),
333
+ "proposal_ids": [str(proposal.id)],
334
+ }
335
+ )
336
+
337
+ return results
backend/app/repositories/rule_repository.py CHANGED
@@ -0,0 +1,23 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ from sqlalchemy.orm import Session
4
+
5
+ from app.models.rule import Rule
6
+
7
+
8
+ class RuleRepository:
9
+ """Repository for workspace-scoped validation rules."""
10
+
11
+ def __init__(self, db: Session):
12
+ self.db = db
13
+
14
+ def list_enabled(self, workspace_id) -> list[Rule]:
15
+ return (
16
+ self.db.query(Rule)
17
+ .filter(
18
+ Rule.workspace_id == workspace_id,
19
+ Rule.enabled.is_(True),
20
+ )
21
+ .order_by(Rule.created_at.asc())
22
+ .all()
23
+ )
backend/app/workflow/graph.py CHANGED
@@ -1,4 +1,4 @@
1
- from __future__ import annotations
2
 
3
  from sqlalchemy.orm import Session
4
  from langgraph.graph import END, START, StateGraph
@@ -7,6 +7,7 @@ from app.llm.client import LLMClient
7
  from app.repositories.knowledge_repository import KnowledgeRepository
8
  from app.services.chunking_service import ChunkingService
9
  from app.services.knowledge_link_service import KnowledgeLinkService
 
10
 
11
  from app.workflow.nodes.chunk import chunk
12
  from app.workflow.nodes.classify import classify
@@ -15,11 +16,19 @@ from app.workflow.nodes.extract import extract
15
  from app.workflow.nodes.knowledge import knowledge
16
  from app.workflow.nodes.link import link
17
  from app.workflow.nodes.reconcile import reconcile
18
- from app.workflow.state import WorkflowState
19
- from app.services.embedding_service import EmbeddingService
20
  from app.workflow.nodes.embed import embed
 
 
 
 
 
21
 
22
- def build_workflow(db: Session, checkpointer=None,interrupt_before=None,):
 
 
 
 
 
23
  """
24
  Build the DocWeave document intelligence workflow.
25
  """
@@ -29,13 +38,17 @@ def build_workflow(db: Session, checkpointer=None,interrupt_before=None,):
29
  chunking_service = ChunkingService(db)
30
  knowledge_link_service = KnowledgeLinkService(db)
31
  embedding_service = EmbeddingService()
 
32
  graph = StateGraph(WorkflowState)
33
 
34
  # ------------------------------------------------------------------
35
  # Nodes
36
  # ------------------------------------------------------------------
37
 
38
- graph.add_node("extract", extract)
 
 
 
39
 
40
  graph.add_node(
41
  "chunk",
@@ -45,7 +58,19 @@ def build_workflow(db: Session, checkpointer=None,interrupt_before=None,):
45
  ),
46
  )
47
 
48
- graph.add_node("classify", classify)
 
 
 
 
 
 
 
 
 
 
 
 
49
 
50
  graph.add_node(
51
  "knowledge",
@@ -64,6 +89,19 @@ def build_workflow(db: Session, checkpointer=None,interrupt_before=None,):
64
  ),
65
  )
66
 
 
 
 
 
 
 
 
 
 
 
 
 
 
67
  graph.add_node(
68
  "link",
69
  lambda state: link(
@@ -72,29 +110,94 @@ def build_workflow(db: Session, checkpointer=None,interrupt_before=None,):
72
  ),
73
  )
74
 
75
- graph.add_node("complete", complete)
 
 
 
 
76
 
77
  graph.add_node(
78
- "embedding",
79
- lambda state: embed(
80
- state,
81
- db,
82
- embedding_service,
83
- ),
84
  )
 
 
 
 
 
 
85
  # ------------------------------------------------------------------
86
  # Workflow edges
87
  # ------------------------------------------------------------------
88
 
89
- graph.add_edge(START, "extract")
90
- graph.add_edge("extract", "chunk")
91
- graph.add_edge("chunk", "embedding")
92
- graph.add_edge("embedding", "classify")
93
- graph.add_edge("classify", "knowledge")
94
- graph.add_edge("knowledge", "reconciliation")
95
- graph.add_edge("reconciliation", "link")
96
- graph.add_edge("link", "complete")
97
- graph.add_edge("complete", END)
98
-
99
-
100
- return graph.compile(checkpointer=checkpointer, interrupt_before=interrupt_before,)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
 
3
  from sqlalchemy.orm import Session
4
  from langgraph.graph import END, START, StateGraph
 
7
  from app.repositories.knowledge_repository import KnowledgeRepository
8
  from app.services.chunking_service import ChunkingService
9
  from app.services.knowledge_link_service import KnowledgeLinkService
10
+ from app.services.embedding_service import EmbeddingService
11
 
12
  from app.workflow.nodes.chunk import chunk
13
  from app.workflow.nodes.classify import classify
 
16
  from app.workflow.nodes.knowledge import knowledge
17
  from app.workflow.nodes.link import link
18
  from app.workflow.nodes.reconcile import reconcile
 
 
19
  from app.workflow.nodes.embed import embed
20
+ from app.workflow.nodes.validate import validate
21
+ from app.workflow.nodes.decision import decide
22
+
23
+ from app.workflow.router import route_after_decision
24
+ from app.workflow.state import WorkflowState
25
 
26
+
27
+ def build_workflow(
28
+ db: Session,
29
+ checkpointer=None,
30
+ interrupt_before=None,
31
+ ):
32
  """
33
  Build the DocWeave document intelligence workflow.
34
  """
 
38
  chunking_service = ChunkingService(db)
39
  knowledge_link_service = KnowledgeLinkService(db)
40
  embedding_service = EmbeddingService()
41
+
42
  graph = StateGraph(WorkflowState)
43
 
44
  # ------------------------------------------------------------------
45
  # Nodes
46
  # ------------------------------------------------------------------
47
 
48
+ graph.add_node(
49
+ "extract",
50
+ extract,
51
+ )
52
 
53
  graph.add_node(
54
  "chunk",
 
58
  ),
59
  )
60
 
61
+ graph.add_node(
62
+ "embedding",
63
+ lambda state: embed(
64
+ state,
65
+ db,
66
+ embedding_service,
67
+ ),
68
+ )
69
+
70
+ graph.add_node(
71
+ "classify",
72
+ classify,
73
+ )
74
 
75
  graph.add_node(
76
  "knowledge",
 
89
  ),
90
  )
91
 
92
+ graph.add_node(
93
+ "validation",
94
+ lambda state: validate(
95
+ state,
96
+ db,
97
+ ),
98
+ )
99
+
100
+ graph.add_node(
101
+ "decision",
102
+ decide,
103
+ )
104
+
105
  graph.add_node(
106
  "link",
107
  lambda state: link(
 
110
  ),
111
  )
112
 
113
+ # Temporary branch target.
114
+ # This will become the real durable human gate later.
115
+ def human_review(state: WorkflowState) -> WorkflowState:
116
+ state.current_node = "HUMAN_REVIEW"
117
+ return state
118
 
119
  graph.add_node(
120
+ "human_review",
121
+ human_review,
 
 
 
 
122
  )
123
+
124
+ graph.add_node(
125
+ "complete",
126
+ complete,
127
+ )
128
+
129
  # ------------------------------------------------------------------
130
  # Workflow edges
131
  # ------------------------------------------------------------------
132
 
133
+ graph.add_edge(
134
+ START,
135
+ "extract",
136
+ )
137
+
138
+ graph.add_edge(
139
+ "extract",
140
+ "chunk",
141
+ )
142
+
143
+ graph.add_edge(
144
+ "chunk",
145
+ "embedding",
146
+ )
147
+
148
+ graph.add_edge(
149
+ "embedding",
150
+ "classify",
151
+ )
152
+
153
+ graph.add_edge(
154
+ "classify",
155
+ "knowledge",
156
+ )
157
+
158
+ graph.add_edge(
159
+ "knowledge",
160
+ "reconciliation",
161
+ )
162
+
163
+ graph.add_edge(
164
+ "reconciliation",
165
+ "validation",
166
+ )
167
+
168
+ graph.add_edge(
169
+ "validation",
170
+ "decision",
171
+ )
172
+
173
+ # Decision controls the workflow path.
174
+ graph.add_conditional_edges(
175
+ "decision",
176
+ route_after_decision,
177
+ {
178
+ "link": "link",
179
+ "human_review": "human_review",
180
+ },
181
+ )
182
+
183
+ graph.add_edge(
184
+ "link",
185
+ "complete",
186
+ )
187
+
188
+ # Temporary review branch returns to completion.
189
+ # We will replace this with the real human gate later.
190
+ graph.add_edge(
191
+ "human_review",
192
+ "complete",
193
+ )
194
+
195
+ graph.add_edge(
196
+ "complete",
197
+ END,
198
+ )
199
+
200
+ return graph.compile(
201
+ checkpointer=checkpointer,
202
+ interrupt_before=interrupt_before,
203
+ )
backend/app/workflow/nodes/__init__.py CHANGED
@@ -1,9 +1,11 @@
1
  from .extract import extract
2
  from .classify import classify
3
  from .complete import complete
 
4
 
5
  __all__ = [
6
  "extract",
7
  "classify",
8
  "complete",
 
9
  ]
 
1
  from .extract import extract
2
  from .classify import classify
3
  from .complete import complete
4
+ from .validate import validate
5
 
6
  __all__ = [
7
  "extract",
8
  "classify",
9
  "complete",
10
+ "validate",
11
  ]
backend/app/workflow/nodes/decision.py ADDED
@@ -0,0 +1,20 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ from app.agents.decision import DecisionAgent
4
+ from app.workflow.state import WorkflowState
5
+
6
+
7
+ def decide(state: WorkflowState) -> WorkflowState:
8
+ """
9
+ Convert validation findings into a workflow decision.
10
+ """
11
+
12
+ state.current_node = "DECISION"
13
+
14
+ agent = DecisionAgent()
15
+
16
+ state.decision = agent.decide(
17
+ state.validation_results
18
+ )
19
+
20
+ return state
backend/app/workflow/nodes/validate.py ADDED
@@ -0,0 +1,99 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ from app.agents.validation import RuleValidationAgent
4
+ from app.models.proposal import Proposal, ProposalType, ProposalStatus
5
+ from app.repositories.knowledge_repository import KnowledgeRepository
6
+ from app.repositories.rule_repository import RuleRepository
7
+ from app.workflow.state import WorkflowState
8
+
9
+
10
+ def validate(
11
+ state: WorkflowState,
12
+ db,
13
+ ) -> WorkflowState:
14
+ """
15
+ Validate proposals generated by reconciliation against
16
+ enabled workspace rules.
17
+ """
18
+
19
+ state.current_node = "RULE_VALIDATION"
20
+
21
+ knowledge_repository = KnowledgeRepository(db)
22
+ rule_repository = RuleRepository(db)
23
+
24
+ new_items = knowledge_repository.list_by_document_version(
25
+ state.document_version_id
26
+ )
27
+
28
+ new_item_ids = {str(item.id) for item in new_items}
29
+
30
+ if not new_item_ids:
31
+ state.validation_results = []
32
+ state.metadata["validation_summary"] = {
33
+ "status": "PASS",
34
+ "rules_evaluated": 0,
35
+ "proposals_evaluated": 0,
36
+ "failures": 0,
37
+ "warnings": 0,
38
+ }
39
+ return state
40
+
41
+ proposals = (
42
+ db.query(Proposal)
43
+ .filter(
44
+ Proposal.workspace_id == state.workspace_id,
45
+ Proposal.status == ProposalStatus.PENDING,
46
+ )
47
+ .all()
48
+ )
49
+
50
+ relevant_proposals = []
51
+
52
+ for proposal in proposals:
53
+ changes = proposal.proposed_changes or {}
54
+
55
+ if proposal.proposal_type == ProposalType.CREATE:
56
+ if proposal.knowledge_item_id is not None:
57
+ if str(proposal.knowledge_item_id) in new_item_ids:
58
+ relevant_proposals.append(proposal)
59
+
60
+ elif proposal.proposal_type == ProposalType.UPDATE:
61
+ source_id = changes.get("source_knowledge_item_id")
62
+
63
+ if source_id and str(source_id) in new_item_ids:
64
+ relevant_proposals.append(proposal)
65
+
66
+ rules = rule_repository.list_enabled(state.workspace_id)
67
+
68
+ agent = RuleValidationAgent()
69
+
70
+ results = agent.validate(
71
+ rules=rules,
72
+ proposals=relevant_proposals,
73
+ )
74
+
75
+ state.validation_results = results
76
+
77
+ failures = sum(
78
+ 1 for result in results if result["status"] == "FAIL"
79
+ )
80
+
81
+ warnings = sum(
82
+ 1 for result in results if result["status"] == "WARNING"
83
+ )
84
+
85
+ state.metadata["validation_summary"] = {
86
+ "status": (
87
+ "FAIL"
88
+ if failures
89
+ else "WARNING"
90
+ if warnings
91
+ else "PASS"
92
+ ),
93
+ "rules_evaluated": len(rules),
94
+ "proposals_evaluated": len(relevant_proposals),
95
+ "failures": failures,
96
+ "warnings": warnings,
97
+ }
98
+
99
+ return state
backend/app/workflow/router.py CHANGED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ from typing import Any
4
+
5
+
6
+ def route_after_decision(state: Any) -> str:
7
+ """
8
+ Route the workflow based on the decision stored in WorkflowState.
9
+ """
10
+
11
+ decision = state.decision.get("decision")
12
+
13
+ if decision == "CONTINUE":
14
+ return "link"
15
+
16
+ if decision == "REVIEW":
17
+ return "human_review"
18
+
19
+ raise ValueError(
20
+ f"Unknown workflow decision: {decision!r}"
21
+ )
backend/app/workflow/state.py CHANGED
@@ -29,6 +29,11 @@ class WorkflowState:
29
 
30
  metadata: dict[str, Any] = field(default_factory=dict)
31
 
 
 
 
 
 
32
  current_node: str = "START"
33
 
34
  completed: bool = False
 
29
 
30
  metadata: dict[str, Any] = field(default_factory=dict)
31
 
32
+ validation_results: list[dict[str, Any]] = field(
33
+ default_factory=list
34
+ )
35
+
36
+ decision: dict[str, Any] = field(default_factory=dict)
37
  current_node: str = "START"
38
 
39
  completed: bool = False
backend/test_decision.py ADDED
@@ -0,0 +1,82 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from app.agents.decision import DecisionAgent
2
+
3
+
4
+ agent = DecisionAgent()
5
+
6
+
7
+ print("\nTEST 1 - ALL PASS")
8
+
9
+ result = agent.decide([
10
+ {
11
+ "status": "PASS",
12
+ "severity": "INFO",
13
+ "proposal_ids": ["proposal-1"],
14
+ }
15
+ ])
16
+
17
+ print(result)
18
+
19
+ assert result["decision"] == "CONTINUE"
20
+
21
+
22
+ print("\nTEST 2 - FAIL")
23
+
24
+ result = agent.decide([
25
+ {
26
+ "status": "FAIL",
27
+ "severity": "HIGH",
28
+ "proposal_ids": ["proposal-2"],
29
+ }
30
+ ])
31
+
32
+ print(result)
33
+
34
+ assert result["decision"] == "REVIEW"
35
+ assert result["failure_count"] == 1
36
+ assert "proposal-2" in result["affected_proposal_ids"]
37
+
38
+
39
+ print("\nTEST 3 - WARNING")
40
+
41
+ result = agent.decide([
42
+ {
43
+ "status": "WARNING",
44
+ "severity": "MEDIUM",
45
+ "proposal_ids": ["proposal-3"],
46
+ }
47
+ ])
48
+
49
+ print(result)
50
+
51
+ assert result["decision"] == "REVIEW"
52
+ assert result["warning_count"] == 1
53
+ assert "proposal-3" in result["affected_proposal_ids"]
54
+
55
+
56
+ print("\nTEST 4 - FAIL + WARNING")
57
+
58
+ result = agent.decide([
59
+ {
60
+ "status": "FAIL",
61
+ "severity": "HIGH",
62
+ "proposal_ids": ["proposal-4"],
63
+ },
64
+ {
65
+ "status": "WARNING",
66
+ "severity": "MEDIUM",
67
+ "proposal_ids": ["proposal-5"],
68
+ }
69
+ ])
70
+
71
+ print(result)
72
+
73
+ assert result["decision"] == "REVIEW"
74
+ assert result["failure_count"] == 1
75
+ assert result["warning_count"] == 1
76
+ assert "proposal-4" in result["affected_proposal_ids"]
77
+ assert "proposal-5" in result["affected_proposal_ids"]
78
+
79
+
80
+ print("\n===================================")
81
+ print("ALL DECISION TESTS PASSED")
82
+ print("===================================")
backend/test_router.py ADDED
@@ -0,0 +1,53 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from types import SimpleNamespace
2
+
3
+ from app.workflow.router import route_after_decision
4
+
5
+
6
+ print("\nTEST 1 - CONTINUE")
7
+
8
+ state = SimpleNamespace(
9
+ decision={
10
+ "decision": "CONTINUE",
11
+ }
12
+ )
13
+
14
+ result = route_after_decision(state)
15
+
16
+ print("ROUTE:", result)
17
+
18
+ assert result == "link"
19
+
20
+
21
+ print("\nTEST 2 - REVIEW")
22
+
23
+ state = SimpleNamespace(
24
+ decision={
25
+ "decision": "REVIEW",
26
+ }
27
+ )
28
+
29
+ result = route_after_decision(state)
30
+
31
+ print("ROUTE:", result)
32
+
33
+ assert result == "human_review"
34
+
35
+
36
+ print("\nTEST 3 - UNKNOWN DECISION")
37
+
38
+ state = SimpleNamespace(
39
+ decision={
40
+ "decision": "SOMETHING_ELSE",
41
+ }
42
+ )
43
+
44
+ try:
45
+ route_after_decision(state)
46
+ raise AssertionError("Expected ValueError")
47
+ except ValueError as exc:
48
+ print("EXPECTED ERROR:", exc)
49
+
50
+
51
+ print("\n===================================")
52
+ print("ALL ROUTER TESTS PASSED")
53
+ print("===================================")
backend/test_validation.py ADDED
@@ -0,0 +1,168 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from types import SimpleNamespace
2
+ from uuid import uuid4
3
+
4
+ from app.agents.validation import RuleValidationAgent
5
+
6
+
7
+ agent = RuleValidationAgent()
8
+
9
+
10
+ def make_rule(name, configuration):
11
+ return SimpleNamespace(
12
+ id=uuid4(),
13
+ name=name,
14
+ description="test rule",
15
+ rule_type="COMPLIANCE",
16
+ configuration=configuration,
17
+ enabled=True,
18
+ )
19
+
20
+
21
+ def make_proposal(confidence=0.90, evidence=True):
22
+ proposed = {
23
+ "confidence": confidence,
24
+ }
25
+
26
+ if evidence:
27
+ proposed["evidence"] = {
28
+ "source": "test.pdf",
29
+ "page": 1,
30
+ }
31
+
32
+ return SimpleNamespace(
33
+ id=uuid4(),
34
+ proposal_type="CREATE",
35
+ proposed_changes=proposed,
36
+ )
37
+
38
+
39
+ # TEST 1: No rules
40
+ result = agent.validate(
41
+ rules=[],
42
+ proposals=[make_proposal()],
43
+ )
44
+
45
+ print("\nTEST 1 - NO RULES")
46
+ print(result)
47
+
48
+
49
+ # TEST 2: Confidence passes
50
+ result = agent.validate(
51
+ rules=[
52
+ make_rule(
53
+ "Minimum confidence",
54
+ {
55
+ "operator": "min_confidence",
56
+ "value": 0.80,
57
+ },
58
+ )
59
+ ],
60
+ proposals=[
61
+ make_proposal(confidence=0.90),
62
+ ],
63
+ )
64
+
65
+ print("\nTEST 2 - CONFIDENCE PASS")
66
+ print(result)
67
+
68
+
69
+ # TEST 3: Confidence fails
70
+ result = agent.validate(
71
+ rules=[
72
+ make_rule(
73
+ "Minimum confidence",
74
+ {
75
+ "operator": "min_confidence",
76
+ "value": 0.80,
77
+ },
78
+ )
79
+ ],
80
+ proposals=[
81
+ make_proposal(confidence=0.50),
82
+ ],
83
+ )
84
+
85
+ print("\nTEST 3 - CONFIDENCE FAIL")
86
+ print(result)
87
+
88
+
89
+ # TEST 4: Evidence passes
90
+ result = agent.validate(
91
+ rules=[
92
+ make_rule(
93
+ "Required evidence",
94
+ {
95
+ "operator": "required_evidence",
96
+ },
97
+ )
98
+ ],
99
+ proposals=[
100
+ make_proposal(evidence=True),
101
+ ],
102
+ )
103
+
104
+ print("\nTEST 4 - EVIDENCE PASS")
105
+ print(result)
106
+
107
+
108
+ # TEST 5: Evidence fails
109
+ result = agent.validate(
110
+ rules=[
111
+ make_rule(
112
+ "Required evidence",
113
+ {
114
+ "operator": "required_evidence",
115
+ },
116
+ )
117
+ ],
118
+ proposals=[
119
+ make_proposal(evidence=False),
120
+ ],
121
+ )
122
+
123
+ print("\nTEST 5 - EVIDENCE FAIL")
124
+ print(result)
125
+
126
+
127
+ # TEST 6: Malformed rule
128
+ result = agent.validate(
129
+ rules=[
130
+ make_rule(
131
+ "Broken confidence rule",
132
+ {
133
+ "operator": "min_confidence",
134
+ "value": "not-a-number",
135
+ },
136
+ )
137
+ ],
138
+ proposals=[
139
+ make_proposal(),
140
+ ],
141
+ )
142
+
143
+ print("\nTEST 6 - MALFORMED RULE")
144
+ print(result)
145
+
146
+
147
+ # TEST 7: Unsupported operator
148
+ result = agent.validate(
149
+ rules=[
150
+ make_rule(
151
+ "Unknown rule",
152
+ {
153
+ "operator": "unsupported_operator",
154
+ },
155
+ )
156
+ ],
157
+ proposals=[
158
+ make_proposal(),
159
+ ],
160
+ )
161
+
162
+ print("\nTEST 7 - UNKNOWN OPERATOR")
163
+ print(result)
164
+
165
+
166
+ print("\n===================================")
167
+ print("VALIDATION TEST RUN COMPLETE")
168
+ print("===================================")