shak3008 commited on
Commit
af21975
·
1 Parent(s): e2532ee

feat: build document intelligence workflow

Browse files
backend/alembic/versions/5b7572e0875a_add_document_chunks.py ADDED
@@ -0,0 +1,41 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """add document chunks
2
+
3
+ Revision ID: 5b7572e0875a
4
+ Revises: 0fd360eb89e4
5
+ Create Date: 2026-08-10 15:17:52.645173
6
+
7
+ """
8
+ from typing import Sequence, Union
9
+
10
+ from alembic import op
11
+ import sqlalchemy as sa
12
+
13
+
14
+ revision: str = '5b7572e0875a'
15
+ down_revision: Union[str, None] = '0fd360eb89e4'
16
+ branch_labels: Union[str, Sequence[str], None] = None
17
+ depends_on: Union[str, Sequence[str], None] = None
18
+
19
+
20
+ def upgrade() -> None:
21
+ # ### commands auto generated by Alembic - please adjust! ###
22
+ op.create_table('document_chunks',
23
+ sa.Column('id', sa.UUID(), nullable=False),
24
+ sa.Column('document_version_id', sa.UUID(), nullable=False),
25
+ sa.Column('chunk_index', sa.Integer(), nullable=False),
26
+ sa.Column('text', sa.Text(), nullable=False),
27
+ sa.Column('page_number', sa.Integer(), nullable=True),
28
+ sa.Column('section', sa.Text(), nullable=True),
29
+ sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
30
+ sa.ForeignKeyConstraint(['document_version_id'], ['document_versions.id'], ondelete='CASCADE'),
31
+ sa.PrimaryKeyConstraint('id')
32
+ )
33
+ op.create_index(op.f('ix_document_chunks_document_version_id'), 'document_chunks', ['document_version_id'], unique=False)
34
+ # ### end Alembic commands ###
35
+
36
+
37
+ def downgrade() -> None:
38
+ # ### commands auto generated by Alembic - please adjust! ###
39
+ op.drop_index(op.f('ix_document_chunks_document_version_id'), table_name='document_chunks')
40
+ op.drop_table('document_chunks')
41
+ # ### end Alembic commands ###
backend/app/models/__init__.py CHANGED
@@ -7,6 +7,7 @@ from .user import User
7
  from .workspace import Workspace
8
  from .document import Document
9
  from .document_version import DocumentVersion
 
10
 
11
  from .knowledge_item import KnowledgeItem
12
  from .knowledge_evidence import KnowledgeEvidence
 
7
  from .workspace import Workspace
8
  from .document import Document
9
  from .document_version import DocumentVersion
10
+ from .document_chunk import DocumentChunk
11
 
12
  from .knowledge_item import KnowledgeItem
13
  from .knowledge_evidence import KnowledgeEvidence
backend/app/models/document_chunk.py ADDED
@@ -0,0 +1,74 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ import uuid
4
+ from datetime import datetime
5
+
6
+ from sqlalchemy import DateTime, ForeignKey, Integer, Text, func
7
+ from sqlalchemy.dialects.postgresql import UUID
8
+ from sqlalchemy.orm import Mapped, mapped_column, relationship
9
+
10
+ from app.database.database import Base
11
+
12
+
13
+ class DocumentChunk(Base):
14
+ """
15
+ A retrievable chunk of a specific document version.
16
+
17
+ Chunks preserve lightweight provenance so retrieval results
18
+ can be traced back to their original document location.
19
+ """
20
+
21
+ __tablename__ = "document_chunks"
22
+
23
+ id: Mapped[uuid.UUID] = mapped_column(
24
+ UUID(as_uuid=True),
25
+ primary_key=True,
26
+ default=uuid.uuid4,
27
+ )
28
+
29
+ document_version_id: Mapped[uuid.UUID] = mapped_column(
30
+ UUID(as_uuid=True),
31
+ ForeignKey(
32
+ "document_versions.id",
33
+ ondelete="CASCADE",
34
+ ),
35
+ nullable=False,
36
+ index=True,
37
+ )
38
+
39
+ chunk_index: Mapped[int] = mapped_column(
40
+ Integer,
41
+ nullable=False,
42
+ )
43
+
44
+ text: Mapped[str] = mapped_column(
45
+ Text,
46
+ nullable=False,
47
+ )
48
+
49
+ page_number: Mapped[int | None] = mapped_column(
50
+ Integer,
51
+ nullable=True,
52
+ )
53
+
54
+ section: Mapped[str | None] = mapped_column(
55
+ Text,
56
+ nullable=True,
57
+ )
58
+
59
+ created_at: Mapped[datetime] = mapped_column(
60
+ DateTime(timezone=True),
61
+ server_default=func.now(),
62
+ nullable=False,
63
+ )
64
+
65
+ document_version: Mapped["DocumentVersion"] = relationship(
66
+ back_populates="chunks",
67
+ )
68
+
69
+ def __repr__(self) -> str:
70
+ return (
71
+ f"<DocumentChunk("
72
+ f"document_version={self.document_version_id}, "
73
+ f"index={self.chunk_index})>"
74
+ )
backend/app/models/document_version.py CHANGED
@@ -18,6 +18,10 @@ from sqlalchemy.orm import Mapped, mapped_column, relationship
18
 
19
  from app.database.database import Base
20
 
 
 
 
 
21
 
22
  class DocumentVersionStatus(str, enum.Enum):
23
  UPLOADED = "UPLOADED"
@@ -129,6 +133,11 @@ class DocumentVersion(Base):
129
  cascade="all, delete-orphan",
130
  )
131
 
 
 
 
 
 
132
  workflow_runs: Mapped[list["WorkflowRun"]] = relationship(
133
  back_populates="document_version",
134
  cascade="all, delete-orphan",
 
18
 
19
  from app.database.database import Base
20
 
21
+ from typing import TYPE_CHECKING
22
+
23
+ if TYPE_CHECKING:
24
+ from app.models.document_chunk import DocumentChunk
25
 
26
  class DocumentVersionStatus(str, enum.Enum):
27
  UPLOADED = "UPLOADED"
 
133
  cascade="all, delete-orphan",
134
  )
135
 
136
+ chunks: Mapped[list["DocumentChunk"]] = relationship(
137
+ back_populates="document_version",
138
+ cascade="all, delete-orphan",
139
+ )
140
+
141
  workflow_runs: Mapped[list["WorkflowRun"]] = relationship(
142
  back_populates="document_version",
143
  cascade="all, delete-orphan",
backend/app/repositories/knowledge_repository.py CHANGED
@@ -79,4 +79,21 @@ class KnowledgeRepository(BaseRepository[KnowledgeItem]):
79
  == document_version_id
80
  )
81
  .all()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
82
  )
 
79
  == document_version_id
80
  )
81
  .all()
82
+ )
83
+
84
+ def find_candidates(
85
+ self,
86
+ *,
87
+ workspace_id,
88
+ knowledge_type,
89
+ title,
90
+ ):
91
+ return (
92
+ self.db.query(KnowledgeItem)
93
+ .filter(
94
+ KnowledgeItem.workspace_id == workspace_id,
95
+ KnowledgeItem.type == knowledge_type,
96
+ KnowledgeItem.title.ilike(f"%{title}%"),
97
+ )
98
+ .all()
99
  )
backend/app/services/chunking_service.py ADDED
@@ -0,0 +1,101 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ from sqlalchemy.orm import Session
4
+
5
+ from app.models.document_chunk import DocumentChunk
6
+
7
+
8
+ class ChunkingService:
9
+ """
10
+ Converts extracted document sections into persistent,
11
+ retrieval-ready document chunks.
12
+ """
13
+
14
+ CHUNK_SIZE = 500
15
+ OVERLAP = 50
16
+
17
+ def __init__(self, db: Session):
18
+ self.db = db
19
+
20
+ def chunk_document(
21
+ self,
22
+ *,
23
+ document_version_id,
24
+ sections: list[dict],
25
+ ) -> dict:
26
+
27
+ # Make chunk generation idempotent.
28
+ # Re-running the workflow won't create duplicate chunks.
29
+ self.db.query(DocumentChunk).filter(
30
+ DocumentChunk.document_version_id
31
+ == document_version_id
32
+ ).delete(
33
+ synchronize_session=False
34
+ )
35
+
36
+ chunk_index = 0
37
+
38
+ for section in sections:
39
+ text = (section.get("text") or "").strip()
40
+
41
+ if not text:
42
+ continue
43
+
44
+ metadata = section.get("metadata") or {}
45
+
46
+ chunks = self._chunk_text(text)
47
+
48
+ for chunk_text in chunks:
49
+ self.db.add(
50
+ DocumentChunk(
51
+ document_version_id=document_version_id,
52
+ chunk_index=chunk_index,
53
+ text=chunk_text,
54
+ page_number=metadata.get("page"),
55
+ section=(
56
+ metadata.get("section_title")
57
+ or metadata.get("section_type")
58
+ ),
59
+ )
60
+ )
61
+
62
+ chunk_index += 1
63
+
64
+ self.db.commit()
65
+
66
+ return {
67
+ "chunks_created": chunk_index,
68
+ }
69
+
70
+ @classmethod
71
+ def _chunk_text(
72
+ cls,
73
+ text: str,
74
+ ) -> list[str]:
75
+
76
+ if len(text) <= cls.CHUNK_SIZE:
77
+ return [text]
78
+
79
+ chunks = []
80
+
81
+ start = 0
82
+ text_length = len(text)
83
+
84
+ while start < text_length:
85
+
86
+ end = min(
87
+ start + cls.CHUNK_SIZE,
88
+ text_length,
89
+ )
90
+
91
+ chunk = text[start:end].strip()
92
+
93
+ if chunk:
94
+ chunks.append(chunk)
95
+
96
+ if end >= text_length:
97
+ break
98
+
99
+ start = end - cls.OVERLAP
100
+
101
+ return chunks
backend/app/services/document_processor.py CHANGED
@@ -107,6 +107,65 @@ def clean_text(text: str) -> str:
107
  text = re.sub(r"\n{3,}", "\n\n", text)
108
  return text.strip()
109
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
110
 
111
  def detect_type(file_path: str, mime_type: str = None):
112
  extension = os.path.splitext(file_path)[1].lower()
@@ -285,7 +344,10 @@ def extract_pdf_docling(file_path: str) -> list[TextSection]:
285
 
286
  def extract_pdf_ocr(file_path: str) -> list[TextSection]:
287
  try:
288
- images = convert_from_path(file_path)
 
 
 
289
  sections = []
290
 
291
  for page_number, image in enumerate(images, start=1):
@@ -305,21 +367,73 @@ def extract_pdf_ocr(file_path: str) -> list[TextSection]:
305
 
306
 
307
  def extract_pdf_sections(file_path: str) -> list[TextSection]:
308
- """PyMuPDF → Docling → OCR cascade."""
309
- sections = extract_pdf_text_pymupdf(file_path) if fitz else extract_pdf_text_pypdf(file_path)
 
 
 
 
 
 
 
310
 
311
- if sum(len(s.text) for s in sections) > 500:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
312
  return sections
313
 
314
- logger.info("PyMuPDF extraction weak, trying Docling")
 
 
 
 
 
 
 
315
  sections = extract_pdf_docling(file_path)
316
 
317
- if sum(len(s.text) for s in sections) > 500:
 
 
 
 
 
 
 
 
318
  return sections
319
 
320
- logger.info("Docling failed, OCR triggered")
321
- return extract_pdf_ocr(file_path)
 
 
 
 
 
322
 
 
 
 
 
 
 
 
 
 
 
 
323
 
324
  # ---------------------------------------------------------------------------
325
  # Other format extractors
 
107
  text = re.sub(r"\n{3,}", "\n\n", text)
108
  return text.strip()
109
 
110
+ def extraction_quality(sections: list[TextSection]) -> float:
111
+ """
112
+ Estimate whether extracted PDF text is substantial enough
113
+ to be considered usable.
114
+
115
+ Returns a score from 0.0 to 1.0.
116
+ """
117
+
118
+ if not sections:
119
+ return 0.0
120
+
121
+ total_chars = sum(len(section.text) for section in sections)
122
+
123
+ if total_chars == 0:
124
+ return 0.0
125
+
126
+ text = "\n".join(section.text for section in sections)
127
+
128
+ words = re.findall(r"\b\w+\b", text)
129
+ if not words:
130
+ return 0.0
131
+
132
+ alphanumeric_chars = sum(
133
+ char.isalnum()
134
+ for char in text
135
+ )
136
+
137
+ alphanumeric_ratio = alphanumeric_chars / max(len(text), 1)
138
+
139
+ word_count = len(words)
140
+
141
+ score = 0.0
142
+
143
+ # Amount of actual text
144
+ if total_chars >= 2000:
145
+ score += 0.4
146
+ elif total_chars >= 1000:
147
+ score += 0.25
148
+ elif total_chars >= 500:
149
+ score += 0.1
150
+
151
+ # Number of words
152
+ if word_count >= 300:
153
+ score += 0.3
154
+ elif word_count >= 150:
155
+ score += 0.2
156
+ elif word_count >= 75:
157
+ score += 0.1
158
+
159
+ # Mostly actual text rather than symbols/noise
160
+ if alphanumeric_ratio >= 0.75:
161
+ score += 0.3
162
+ elif alphanumeric_ratio >= 0.60:
163
+ score += 0.2
164
+ elif alphanumeric_ratio >= 0.45:
165
+ score += 0.1
166
+
167
+ return min(score, 1.0)
168
+
169
 
170
  def detect_type(file_path: str, mime_type: str = None):
171
  extension = os.path.splitext(file_path)[1].lower()
 
344
 
345
  def extract_pdf_ocr(file_path: str) -> list[TextSection]:
346
  try:
347
+ images = convert_from_path(
348
+ file_path,
349
+ poppler_path=os.getenv("POPPLER_PATH"),
350
+ )
351
  sections = []
352
 
353
  for page_number, image in enumerate(images, start=1):
 
367
 
368
 
369
  def extract_pdf_sections(file_path: str) -> list[TextSection]:
370
+ """
371
+ Extract PDF text using a quality-aware cascade:
372
+
373
+ PyMuPDF → Docling → OCR
374
+ """
375
+
376
+ # ------------------------------------------------------------
377
+ # 1. PyMuPDF
378
+ # ------------------------------------------------------------
379
 
380
+ sections = (
381
+ extract_pdf_text_pymupdf(file_path)
382
+ if fitz
383
+ else extract_pdf_text_pypdf(file_path)
384
+ )
385
+
386
+ score = extraction_quality(sections)
387
+
388
+ logger.info(
389
+ "PyMuPDF extraction quality: %.2f (%s chars)",
390
+ score,
391
+ sum(len(s.text) for s in sections),
392
+ )
393
+
394
+ if score >= 0.6:
395
  return sections
396
 
397
+ # ------------------------------------------------------------
398
+ # 2. Docling
399
+ # ------------------------------------------------------------
400
+
401
+ logger.info(
402
+ "PyMuPDF extraction quality insufficient, trying Docling"
403
+ )
404
+
405
  sections = extract_pdf_docling(file_path)
406
 
407
+ score = extraction_quality(sections)
408
+
409
+ logger.info(
410
+ "Docling extraction quality: %.2f (%s chars)",
411
+ score,
412
+ sum(len(s.text) for s in sections),
413
+ )
414
+
415
+ if score >= 0.6:
416
  return sections
417
 
418
+ # ------------------------------------------------------------
419
+ # 3. OCR
420
+ # ------------------------------------------------------------
421
+
422
+ logger.info(
423
+ "Docling extraction quality insufficient, triggering OCR"
424
+ )
425
 
426
+ sections = extract_pdf_ocr(file_path)
427
+
428
+ score = extraction_quality(sections)
429
+
430
+ logger.info(
431
+ "OCR extraction quality: %.2f (%s chars)",
432
+ score,
433
+ sum(len(s.text) for s in sections),
434
+ )
435
+
436
+ return sections
437
 
438
  # ---------------------------------------------------------------------------
439
  # Other format extractors
backend/app/services/knowledge_link_service.py ADDED
@@ -0,0 +1,172 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ from difflib import SequenceMatcher
4
+
5
+ from sqlalchemy.orm import Session
6
+
7
+ from app.models.knowledge_item import KnowledgeItem
8
+ from app.models.knowledge_link import (
9
+ KnowledgeLink,
10
+ RelationshipType,
11
+ )
12
+
13
+
14
+ class KnowledgeLinkService:
15
+ """
16
+ Builds semantic relationships between KnowledgeItems.
17
+
18
+ Link creation is intentionally conservative:
19
+ only strong, deterministic relationships are created.
20
+ """
21
+
22
+ def __init__(self, db: Session):
23
+ self.db = db
24
+
25
+ def link_document(
26
+ self,
27
+ *,
28
+ workspace_id,
29
+ document_version_id,
30
+ ) -> dict:
31
+ current_items = (
32
+ self.db.query(KnowledgeItem)
33
+ .filter(
34
+ KnowledgeItem.workspace_id == workspace_id,
35
+ KnowledgeItem.document_version_id == document_version_id,
36
+ )
37
+ .all()
38
+ )
39
+
40
+ existing_items = (
41
+ self.db.query(KnowledgeItem)
42
+ .filter(
43
+ KnowledgeItem.workspace_id == workspace_id,
44
+ KnowledgeItem.document_version_id != document_version_id,
45
+ )
46
+ .all()
47
+ )
48
+
49
+ created = 0
50
+
51
+ for source in current_items:
52
+ for target in existing_items:
53
+ if source.id == target.id:
54
+ continue
55
+
56
+ relationship = self._infer_relationship(
57
+ source,
58
+ target,
59
+ )
60
+
61
+ if relationship is None:
62
+ continue
63
+
64
+ relationship_type, confidence = relationship
65
+
66
+ if self._link_exists(
67
+ source.id,
68
+ target.id,
69
+ relationship_type,
70
+ ):
71
+ continue
72
+
73
+ self.db.add(
74
+ KnowledgeLink(
75
+ source_item_id=source.id,
76
+ target_item_id=target.id,
77
+ relationship_type=relationship_type,
78
+ confidence=confidence,
79
+ )
80
+ )
81
+
82
+ created += 1
83
+
84
+ self.db.commit()
85
+
86
+ return {
87
+ "links_created": created,
88
+ }
89
+
90
+ def _infer_relationship(
91
+ self,
92
+ source: KnowledgeItem,
93
+ target: KnowledgeItem,
94
+ ):
95
+ title_similarity = self._similarity(
96
+ source.title,
97
+ target.title,
98
+ )
99
+
100
+ value_similarity = self._similarity(
101
+ source.value,
102
+ target.value,
103
+ )
104
+
105
+ # Same concept with highly similar values.
106
+ if (
107
+ source.type == target.type
108
+ and title_similarity >= 0.85
109
+ and value_similarity >= 0.80
110
+ ):
111
+ return (
112
+ RelationshipType.RELATED_TO,
113
+ round(
114
+ (title_similarity + value_similarity) / 2,
115
+ 3,
116
+ ),
117
+ )
118
+
119
+ # A method/treatment can be related to an entity/claim.
120
+ if (
121
+ source.type.name == "METHOD"
122
+ and target.type.name in {"ENTITY", "CLAIM", "OBSERVATION"}
123
+ and title_similarity >= 0.65
124
+ ):
125
+ return (
126
+ RelationshipType.USES,
127
+ round(title_similarity, 3),
128
+ )
129
+
130
+ # Metrics commonly describe claims/observations.
131
+ if (
132
+ source.type.name == "METRIC"
133
+ and target.type.name in {"CLAIM", "OBSERVATION"}
134
+ and title_similarity >= 0.65
135
+ ):
136
+ return (
137
+ RelationshipType.REFERENCES,
138
+ round(title_similarity, 3),
139
+ )
140
+
141
+ return None
142
+
143
+ def _link_exists(
144
+ self,
145
+ source_item_id,
146
+ target_item_id,
147
+ relationship_type: RelationshipType,
148
+ ) -> bool:
149
+ return (
150
+ self.db.query(KnowledgeLink)
151
+ .filter(
152
+ KnowledgeLink.source_item_id == source_item_id,
153
+ KnowledgeLink.target_item_id == target_item_id,
154
+ KnowledgeLink.relationship_type == relationship_type,
155
+ )
156
+ .first()
157
+ is not None
158
+ )
159
+
160
+ @staticmethod
161
+ def _similarity(
162
+ left: str | None,
163
+ right: str | None,
164
+ ) -> float:
165
+ if not left or not right:
166
+ return 0.0
167
+
168
+ return SequenceMatcher(
169
+ None,
170
+ left.lower().strip(),
171
+ right.lower().strip(),
172
+ ).ratio()
backend/app/services/reconciliation_service.py ADDED
@@ -0,0 +1,223 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ from sqlalchemy.orm import Session
4
+
5
+ from app.models.knowledge_item import KnowledgeItem
6
+ from app.models.proposal import Proposal, ProposalStatus, ProposalType
7
+ from app.repositories.knowledge_repository import KnowledgeRepository
8
+ from difflib import SequenceMatcher
9
+
10
+ class ReconciliationService:
11
+ """
12
+ Compares newly extracted knowledge against the existing
13
+ Knowledge Register and creates proposals for changes.
14
+
15
+ Reconciliation never directly mutates existing knowledge.
16
+ All changes go through PENDING proposals.
17
+ """
18
+
19
+ def __init__(self, db: Session):
20
+ self.db = db
21
+ self.repository = KnowledgeRepository(db)
22
+
23
+ def reconcile_document(
24
+ self,
25
+ *,
26
+ workspace_id,
27
+ document_version_id,
28
+ ) -> dict:
29
+ new_items = self.repository.list_by_document_version(
30
+ document_version_id
31
+ )
32
+
33
+ existing_items = (
34
+ self.db.query(KnowledgeItem)
35
+ .filter(
36
+ KnowledgeItem.workspace_id == workspace_id,
37
+ KnowledgeItem.document_version_id != document_version_id,
38
+ )
39
+ .all()
40
+ )
41
+
42
+ new_count = 0
43
+ duplicate_count = 0
44
+ conflict_count = 0
45
+
46
+ for item in new_items:
47
+ match = self._find_match(item, existing_items)
48
+
49
+ if match is None:
50
+ self._create_create_proposal(
51
+ workspace_id=workspace_id,
52
+ item=item,
53
+ )
54
+ new_count += 1
55
+ continue
56
+
57
+ if self._is_duplicate(item, match):
58
+ duplicate_count += 1
59
+ continue
60
+
61
+ self._create_update_proposal(
62
+ workspace_id=workspace_id,
63
+ existing_item=match,
64
+ new_item=item,
65
+ )
66
+ conflict_count += 1
67
+
68
+ self.db.commit()
69
+
70
+ return {
71
+ "new": new_count,
72
+ "duplicates": duplicate_count,
73
+ "conflicts": conflict_count,
74
+ }
75
+
76
+ def _find_match(
77
+ self,
78
+ item: KnowledgeItem,
79
+ existing_items: list[KnowledgeItem],
80
+ ) -> KnowledgeItem | None:
81
+
82
+ best_match = None
83
+ best_score = 0.0
84
+
85
+ new_title = self._normalize(item.title)
86
+
87
+ for existing in existing_items:
88
+ if existing.type != item.type:
89
+ continue
90
+
91
+ existing_title = self._normalize(existing.title)
92
+
93
+ # Exact match
94
+ if new_title == existing_title:
95
+ return existing
96
+
97
+ # Similarity match
98
+ score = SequenceMatcher(
99
+ None,
100
+ new_title,
101
+ existing_title,
102
+ ).ratio()
103
+
104
+ if score > best_score:
105
+ best_score = score
106
+ best_match = existing
107
+
108
+ if best_score >= 0.80:
109
+ return best_match
110
+
111
+ return None
112
+
113
+ def _is_duplicate(
114
+ self,
115
+ new_item: KnowledgeItem,
116
+ existing_item: KnowledgeItem,
117
+ ) -> bool:
118
+
119
+ return (
120
+ self._normalize(new_item.value)
121
+ == self._normalize(existing_item.value)
122
+ )
123
+
124
+ def _pending_proposal_exists(
125
+ self,
126
+ knowledge_item_id,
127
+ proposal_type: ProposalType,
128
+ ) -> bool:
129
+
130
+ return (
131
+ self.db.query(Proposal)
132
+ .filter(
133
+ Proposal.knowledge_item_id == knowledge_item_id,
134
+ Proposal.proposal_type == proposal_type,
135
+ Proposal.status == ProposalStatus.PENDING,
136
+ )
137
+ .first()
138
+ is not None
139
+ )
140
+
141
+ def _create_create_proposal(
142
+ self,
143
+ *,
144
+ workspace_id,
145
+ item: KnowledgeItem,
146
+ ):
147
+ if self._pending_proposal_exists(
148
+ item.id,
149
+ ProposalType.CREATE,
150
+ ):
151
+ return
152
+
153
+ proposal = Proposal(
154
+ workspace_id=workspace_id,
155
+ knowledge_item_id=item.id,
156
+ proposal_type=ProposalType.CREATE,
157
+ status=ProposalStatus.PENDING,
158
+ summary=f"Create knowledge item: {item.title}",
159
+ rationale=(
160
+ "No matching knowledge item was found in the "
161
+ "existing Knowledge Register."
162
+ ),
163
+ proposed_changes={
164
+ "type": item.type.value,
165
+ "title": item.title,
166
+ "value": item.value,
167
+ "summary": item.summary,
168
+ "attributes": item.attributes,
169
+ "confidence": item.confidence,
170
+ },
171
+ )
172
+
173
+ self.db.add(proposal)
174
+
175
+ def _create_update_proposal(
176
+ self,
177
+ *,
178
+ workspace_id,
179
+ existing_item: KnowledgeItem,
180
+ new_item: KnowledgeItem,
181
+ ):
182
+ if self._pending_proposal_exists(
183
+ existing_item.id,
184
+ ProposalType.UPDATE,
185
+ ):
186
+ return
187
+
188
+ proposal = Proposal(
189
+ workspace_id=workspace_id,
190
+ knowledge_item_id=existing_item.id,
191
+ proposal_type=ProposalType.UPDATE,
192
+ status=ProposalStatus.PENDING,
193
+ summary=f"Update knowledge item: {existing_item.title}",
194
+ rationale=(
195
+ "A matching knowledge item exists, but the newly "
196
+ "extracted value differs from the registered value."
197
+ ),
198
+ proposed_changes={
199
+ "existing": {
200
+ "value": existing_item.value,
201
+ "summary": existing_item.summary,
202
+ "attributes": existing_item.attributes,
203
+ "confidence": existing_item.confidence,
204
+ },
205
+ "proposed": {
206
+ "value": new_item.value,
207
+ "summary": new_item.summary,
208
+ "attributes": new_item.attributes,
209
+ "confidence": new_item.confidence,
210
+ },
211
+ "source_knowledge_item_id": str(new_item.id),
212
+ },
213
+ )
214
+
215
+ self.db.add(proposal)
216
+ @staticmethod
217
+ def _normalize(value: str | None) -> str:
218
+ if not value:
219
+ return ""
220
+
221
+ words = value.lower().strip().split()
222
+
223
+ return " ".join(sorted(words))
backend/app/workflow/executor.py CHANGED
@@ -2,25 +2,19 @@ from __future__ import annotations
2
 
3
  from sqlalchemy.orm import Session
4
 
5
- from app.llm.client import LLMClient
6
- from app.repositories.knowledge_repository import KnowledgeRepository
7
  from app.services.workflow_service import WorkflowService
8
- from app.workflow.nodes.classify import classify
9
- from app.workflow.nodes.complete import complete
10
- from app.workflow.nodes.extract import extract
11
- from app.workflow.nodes.knowledge import knowledge
12
  from app.workflow.state import WorkflowState
13
 
14
 
15
  class WorkflowExecutor:
16
  """
17
- Executes the document workflow.
18
  """
19
 
20
  def __init__(self, db: Session):
21
  self.workflow_service = WorkflowService(db)
22
- self.llm_client = LLMClient()
23
- self.knowledge_repository = KnowledgeRepository(db)
24
 
25
  def execute(
26
  self,
@@ -29,21 +23,12 @@ class WorkflowExecutor:
29
  ) -> WorkflowState:
30
 
31
  try:
32
- state = extract(state)
33
-
34
- state = classify(state)
35
-
36
- state = knowledge(
37
- state,
38
- self.llm_client,
39
- self.knowledge_repository,
40
- )
41
-
42
- state = complete(state)
43
 
44
  self.workflow_service.complete_workflow(workflow)
45
 
46
- return state
47
 
48
  except Exception:
 
49
  raise
 
2
 
3
  from sqlalchemy.orm import Session
4
 
 
 
5
  from app.services.workflow_service import WorkflowService
6
+ from app.workflow.graph import build_workflow
 
 
 
7
  from app.workflow.state import WorkflowState
8
 
9
 
10
  class WorkflowExecutor:
11
  """
12
+ Executes the document workflow through LangGraph.
13
  """
14
 
15
  def __init__(self, db: Session):
16
  self.workflow_service = WorkflowService(db)
17
+ self.graph = build_workflow(db)
 
18
 
19
  def execute(
20
  self,
 
23
  ) -> WorkflowState:
24
 
25
  try:
26
+ final_state = self.graph.invoke(state)
 
 
 
 
 
 
 
 
 
 
27
 
28
  self.workflow_service.complete_workflow(workflow)
29
 
30
+ return final_state
31
 
32
  except Exception:
33
+ self.workflow_service.fail_workflow(workflow)
34
  raise
backend/app/workflow/graph.py CHANGED
@@ -0,0 +1,89 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ from sqlalchemy.orm import Session
4
+ from langgraph.graph import END, START, StateGraph
5
+
6
+ from app.llm.client import LLMClient
7
+ from app.repositories.knowledge_repository import KnowledgeRepository
8
+ from app.services.chunking_service import ChunkingService
9
+ from app.services.knowledge_link_service import KnowledgeLinkService
10
+
11
+ from app.workflow.nodes.chunk import chunk
12
+ from app.workflow.nodes.classify import classify
13
+ from app.workflow.nodes.complete import complete
14
+ from app.workflow.nodes.extract import extract
15
+ from app.workflow.nodes.knowledge import knowledge
16
+ from app.workflow.nodes.link import link
17
+ from app.workflow.nodes.reconcile import reconcile
18
+ from app.workflow.state import WorkflowState
19
+
20
+
21
+ def build_workflow(db: Session):
22
+ """
23
+ Build the DocWeave document intelligence workflow.
24
+ """
25
+
26
+ llm_client = LLMClient()
27
+ knowledge_repository = KnowledgeRepository(db)
28
+ chunking_service = ChunkingService(db)
29
+ knowledge_link_service = KnowledgeLinkService(db)
30
+
31
+ graph = StateGraph(WorkflowState)
32
+
33
+ # ------------------------------------------------------------------
34
+ # Nodes
35
+ # ------------------------------------------------------------------
36
+
37
+ graph.add_node("extract", extract)
38
+
39
+ graph.add_node(
40
+ "chunk",
41
+ lambda state: chunk(
42
+ state,
43
+ chunking_service,
44
+ ),
45
+ )
46
+
47
+ graph.add_node("classify", classify)
48
+
49
+ graph.add_node(
50
+ "knowledge",
51
+ lambda state: knowledge(
52
+ state,
53
+ llm_client,
54
+ knowledge_repository,
55
+ ),
56
+ )
57
+
58
+ graph.add_node(
59
+ "reconciliation",
60
+ lambda state: reconcile(
61
+ state,
62
+ db,
63
+ ),
64
+ )
65
+
66
+ graph.add_node(
67
+ "link",
68
+ lambda state: link(
69
+ state,
70
+ knowledge_link_service,
71
+ ),
72
+ )
73
+
74
+ graph.add_node("complete", complete)
75
+
76
+ # ------------------------------------------------------------------
77
+ # Workflow edges
78
+ # ------------------------------------------------------------------
79
+
80
+ graph.add_edge(START, "extract")
81
+ graph.add_edge("extract", "chunk")
82
+ graph.add_edge("chunk", "classify")
83
+ graph.add_edge("classify", "knowledge")
84
+ graph.add_edge("knowledge", "reconciliation")
85
+ graph.add_edge("reconciliation", "link")
86
+ graph.add_edge("link", "complete")
87
+ graph.add_edge("complete", END)
88
+
89
+ return graph.compile()
backend/app/workflow/nodes/chunk.py ADDED
@@ -0,0 +1,23 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from app.services.chunking_service import ChunkingService
2
+ from app.workflow.state import WorkflowState
3
+
4
+
5
+ def chunk(
6
+ state: WorkflowState,
7
+ chunking_service: ChunkingService,
8
+ ) -> WorkflowState:
9
+ """
10
+ Persist retrieval-ready document chunks while preserving
11
+ lightweight provenance.
12
+ """
13
+
14
+ state.current_node = "CHUNKING"
15
+
16
+ result = chunking_service.chunk_document(
17
+ document_version_id=state.document_version_id,
18
+ sections=state.extracted_sections,
19
+ )
20
+
21
+ state.metadata["chunks_created"] = result["chunks_created"]
22
+
23
+ return state
backend/app/workflow/nodes/link.py ADDED
@@ -0,0 +1,25 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from app.services.knowledge_link_service import KnowledgeLinkService
2
+ from app.workflow.state import WorkflowState
3
+
4
+
5
+ def link(
6
+ state: WorkflowState,
7
+ knowledge_link_service: KnowledgeLinkService,
8
+ ) -> WorkflowState:
9
+ """
10
+ Create semantic links between the newly extracted knowledge
11
+ and the existing Knowledge Register.
12
+ """
13
+
14
+ state.current_node = "KNOWLEDGE_LINKING"
15
+
16
+ result = knowledge_link_service.link_document(
17
+ workspace_id=state.workspace_id,
18
+ document_version_id=state.document_version_id,
19
+ )
20
+
21
+ state.metadata["knowledge_links_created"] = (
22
+ result["links_created"]
23
+ )
24
+
25
+ return state
backend/app/workflow/nodes/reconcile.py ADDED
@@ -0,0 +1,29 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ from sqlalchemy.orm import Session
4
+
5
+ from app.services.reconciliation_service import ReconciliationService
6
+ from app.workflow.state import WorkflowState
7
+
8
+
9
+ def reconcile(
10
+ state: WorkflowState,
11
+ db: Session,
12
+ ) -> WorkflowState:
13
+ """
14
+ Reconcile newly extracted knowledge against the
15
+ existing Knowledge Register.
16
+ """
17
+
18
+ state.current_node = "RECONCILIATION"
19
+
20
+ service = ReconciliationService(db)
21
+
22
+ results = service.reconcile_document(
23
+ workspace_id=state.workspace_id,
24
+ document_version_id=state.document_version_id,
25
+ )
26
+
27
+ state.metadata["reconciliation"] = results
28
+
29
+ return state