feat: build document intelligence workflow
Browse files- backend/alembic/versions/5b7572e0875a_add_document_chunks.py +41 -0
- backend/app/models/__init__.py +1 -0
- backend/app/models/document_chunk.py +74 -0
- backend/app/models/document_version.py +9 -0
- backend/app/repositories/knowledge_repository.py +17 -0
- backend/app/services/chunking_service.py +101 -0
- backend/app/services/document_processor.py +122 -8
- backend/app/services/knowledge_link_service.py +172 -0
- backend/app/services/reconciliation_service.py +223 -0
- backend/app/workflow/executor.py +6 -21
- backend/app/workflow/graph.py +89 -0
- backend/app/workflow/nodes/chunk.py +23 -0
- backend/app/workflow/nodes/link.py +25 -0
- backend/app/workflow/nodes/reconcile.py +29 -0
backend/alembic/versions/5b7572e0875a_add_document_chunks.py
ADDED
|
@@ -0,0 +1,41 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""add document chunks
|
| 2 |
+
|
| 3 |
+
Revision ID: 5b7572e0875a
|
| 4 |
+
Revises: 0fd360eb89e4
|
| 5 |
+
Create Date: 2026-08-10 15:17:52.645173
|
| 6 |
+
|
| 7 |
+
"""
|
| 8 |
+
from typing import Sequence, Union
|
| 9 |
+
|
| 10 |
+
from alembic import op
|
| 11 |
+
import sqlalchemy as sa
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
revision: str = '5b7572e0875a'
|
| 15 |
+
down_revision: Union[str, None] = '0fd360eb89e4'
|
| 16 |
+
branch_labels: Union[str, Sequence[str], None] = None
|
| 17 |
+
depends_on: Union[str, Sequence[str], None] = None
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
def upgrade() -> None:
|
| 21 |
+
# ### commands auto generated by Alembic - please adjust! ###
|
| 22 |
+
op.create_table('document_chunks',
|
| 23 |
+
sa.Column('id', sa.UUID(), nullable=False),
|
| 24 |
+
sa.Column('document_version_id', sa.UUID(), nullable=False),
|
| 25 |
+
sa.Column('chunk_index', sa.Integer(), nullable=False),
|
| 26 |
+
sa.Column('text', sa.Text(), nullable=False),
|
| 27 |
+
sa.Column('page_number', sa.Integer(), nullable=True),
|
| 28 |
+
sa.Column('section', sa.Text(), nullable=True),
|
| 29 |
+
sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
| 30 |
+
sa.ForeignKeyConstraint(['document_version_id'], ['document_versions.id'], ondelete='CASCADE'),
|
| 31 |
+
sa.PrimaryKeyConstraint('id')
|
| 32 |
+
)
|
| 33 |
+
op.create_index(op.f('ix_document_chunks_document_version_id'), 'document_chunks', ['document_version_id'], unique=False)
|
| 34 |
+
# ### end Alembic commands ###
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
def downgrade() -> None:
|
| 38 |
+
# ### commands auto generated by Alembic - please adjust! ###
|
| 39 |
+
op.drop_index(op.f('ix_document_chunks_document_version_id'), table_name='document_chunks')
|
| 40 |
+
op.drop_table('document_chunks')
|
| 41 |
+
# ### end Alembic commands ###
|
backend/app/models/__init__.py
CHANGED
|
@@ -7,6 +7,7 @@ from .user import User
|
|
| 7 |
from .workspace import Workspace
|
| 8 |
from .document import Document
|
| 9 |
from .document_version import DocumentVersion
|
|
|
|
| 10 |
|
| 11 |
from .knowledge_item import KnowledgeItem
|
| 12 |
from .knowledge_evidence import KnowledgeEvidence
|
|
|
|
| 7 |
from .workspace import Workspace
|
| 8 |
from .document import Document
|
| 9 |
from .document_version import DocumentVersion
|
| 10 |
+
from .document_chunk import DocumentChunk
|
| 11 |
|
| 12 |
from .knowledge_item import KnowledgeItem
|
| 13 |
from .knowledge_evidence import KnowledgeEvidence
|
backend/app/models/document_chunk.py
ADDED
|
@@ -0,0 +1,74 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import uuid
|
| 4 |
+
from datetime import datetime
|
| 5 |
+
|
| 6 |
+
from sqlalchemy import DateTime, ForeignKey, Integer, Text, func
|
| 7 |
+
from sqlalchemy.dialects.postgresql import UUID
|
| 8 |
+
from sqlalchemy.orm import Mapped, mapped_column, relationship
|
| 9 |
+
|
| 10 |
+
from app.database.database import Base
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
class DocumentChunk(Base):
|
| 14 |
+
"""
|
| 15 |
+
A retrievable chunk of a specific document version.
|
| 16 |
+
|
| 17 |
+
Chunks preserve lightweight provenance so retrieval results
|
| 18 |
+
can be traced back to their original document location.
|
| 19 |
+
"""
|
| 20 |
+
|
| 21 |
+
__tablename__ = "document_chunks"
|
| 22 |
+
|
| 23 |
+
id: Mapped[uuid.UUID] = mapped_column(
|
| 24 |
+
UUID(as_uuid=True),
|
| 25 |
+
primary_key=True,
|
| 26 |
+
default=uuid.uuid4,
|
| 27 |
+
)
|
| 28 |
+
|
| 29 |
+
document_version_id: Mapped[uuid.UUID] = mapped_column(
|
| 30 |
+
UUID(as_uuid=True),
|
| 31 |
+
ForeignKey(
|
| 32 |
+
"document_versions.id",
|
| 33 |
+
ondelete="CASCADE",
|
| 34 |
+
),
|
| 35 |
+
nullable=False,
|
| 36 |
+
index=True,
|
| 37 |
+
)
|
| 38 |
+
|
| 39 |
+
chunk_index: Mapped[int] = mapped_column(
|
| 40 |
+
Integer,
|
| 41 |
+
nullable=False,
|
| 42 |
+
)
|
| 43 |
+
|
| 44 |
+
text: Mapped[str] = mapped_column(
|
| 45 |
+
Text,
|
| 46 |
+
nullable=False,
|
| 47 |
+
)
|
| 48 |
+
|
| 49 |
+
page_number: Mapped[int | None] = mapped_column(
|
| 50 |
+
Integer,
|
| 51 |
+
nullable=True,
|
| 52 |
+
)
|
| 53 |
+
|
| 54 |
+
section: Mapped[str | None] = mapped_column(
|
| 55 |
+
Text,
|
| 56 |
+
nullable=True,
|
| 57 |
+
)
|
| 58 |
+
|
| 59 |
+
created_at: Mapped[datetime] = mapped_column(
|
| 60 |
+
DateTime(timezone=True),
|
| 61 |
+
server_default=func.now(),
|
| 62 |
+
nullable=False,
|
| 63 |
+
)
|
| 64 |
+
|
| 65 |
+
document_version: Mapped["DocumentVersion"] = relationship(
|
| 66 |
+
back_populates="chunks",
|
| 67 |
+
)
|
| 68 |
+
|
| 69 |
+
def __repr__(self) -> str:
|
| 70 |
+
return (
|
| 71 |
+
f"<DocumentChunk("
|
| 72 |
+
f"document_version={self.document_version_id}, "
|
| 73 |
+
f"index={self.chunk_index})>"
|
| 74 |
+
)
|
backend/app/models/document_version.py
CHANGED
|
@@ -18,6 +18,10 @@ from sqlalchemy.orm import Mapped, mapped_column, relationship
|
|
| 18 |
|
| 19 |
from app.database.database import Base
|
| 20 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 21 |
|
| 22 |
class DocumentVersionStatus(str, enum.Enum):
|
| 23 |
UPLOADED = "UPLOADED"
|
|
@@ -129,6 +133,11 @@ class DocumentVersion(Base):
|
|
| 129 |
cascade="all, delete-orphan",
|
| 130 |
)
|
| 131 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 132 |
workflow_runs: Mapped[list["WorkflowRun"]] = relationship(
|
| 133 |
back_populates="document_version",
|
| 134 |
cascade="all, delete-orphan",
|
|
|
|
| 18 |
|
| 19 |
from app.database.database import Base
|
| 20 |
|
| 21 |
+
from typing import TYPE_CHECKING
|
| 22 |
+
|
| 23 |
+
if TYPE_CHECKING:
|
| 24 |
+
from app.models.document_chunk import DocumentChunk
|
| 25 |
|
| 26 |
class DocumentVersionStatus(str, enum.Enum):
|
| 27 |
UPLOADED = "UPLOADED"
|
|
|
|
| 133 |
cascade="all, delete-orphan",
|
| 134 |
)
|
| 135 |
|
| 136 |
+
chunks: Mapped[list["DocumentChunk"]] = relationship(
|
| 137 |
+
back_populates="document_version",
|
| 138 |
+
cascade="all, delete-orphan",
|
| 139 |
+
)
|
| 140 |
+
|
| 141 |
workflow_runs: Mapped[list["WorkflowRun"]] = relationship(
|
| 142 |
back_populates="document_version",
|
| 143 |
cascade="all, delete-orphan",
|
backend/app/repositories/knowledge_repository.py
CHANGED
|
@@ -79,4 +79,21 @@ class KnowledgeRepository(BaseRepository[KnowledgeItem]):
|
|
| 79 |
== document_version_id
|
| 80 |
)
|
| 81 |
.all()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 82 |
)
|
|
|
|
| 79 |
== document_version_id
|
| 80 |
)
|
| 81 |
.all()
|
| 82 |
+
)
|
| 83 |
+
|
| 84 |
+
def find_candidates(
|
| 85 |
+
self,
|
| 86 |
+
*,
|
| 87 |
+
workspace_id,
|
| 88 |
+
knowledge_type,
|
| 89 |
+
title,
|
| 90 |
+
):
|
| 91 |
+
return (
|
| 92 |
+
self.db.query(KnowledgeItem)
|
| 93 |
+
.filter(
|
| 94 |
+
KnowledgeItem.workspace_id == workspace_id,
|
| 95 |
+
KnowledgeItem.type == knowledge_type,
|
| 96 |
+
KnowledgeItem.title.ilike(f"%{title}%"),
|
| 97 |
+
)
|
| 98 |
+
.all()
|
| 99 |
)
|
backend/app/services/chunking_service.py
ADDED
|
@@ -0,0 +1,101 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
from sqlalchemy.orm import Session
|
| 4 |
+
|
| 5 |
+
from app.models.document_chunk import DocumentChunk
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
class ChunkingService:
|
| 9 |
+
"""
|
| 10 |
+
Converts extracted document sections into persistent,
|
| 11 |
+
retrieval-ready document chunks.
|
| 12 |
+
"""
|
| 13 |
+
|
| 14 |
+
CHUNK_SIZE = 500
|
| 15 |
+
OVERLAP = 50
|
| 16 |
+
|
| 17 |
+
def __init__(self, db: Session):
|
| 18 |
+
self.db = db
|
| 19 |
+
|
| 20 |
+
def chunk_document(
|
| 21 |
+
self,
|
| 22 |
+
*,
|
| 23 |
+
document_version_id,
|
| 24 |
+
sections: list[dict],
|
| 25 |
+
) -> dict:
|
| 26 |
+
|
| 27 |
+
# Make chunk generation idempotent.
|
| 28 |
+
# Re-running the workflow won't create duplicate chunks.
|
| 29 |
+
self.db.query(DocumentChunk).filter(
|
| 30 |
+
DocumentChunk.document_version_id
|
| 31 |
+
== document_version_id
|
| 32 |
+
).delete(
|
| 33 |
+
synchronize_session=False
|
| 34 |
+
)
|
| 35 |
+
|
| 36 |
+
chunk_index = 0
|
| 37 |
+
|
| 38 |
+
for section in sections:
|
| 39 |
+
text = (section.get("text") or "").strip()
|
| 40 |
+
|
| 41 |
+
if not text:
|
| 42 |
+
continue
|
| 43 |
+
|
| 44 |
+
metadata = section.get("metadata") or {}
|
| 45 |
+
|
| 46 |
+
chunks = self._chunk_text(text)
|
| 47 |
+
|
| 48 |
+
for chunk_text in chunks:
|
| 49 |
+
self.db.add(
|
| 50 |
+
DocumentChunk(
|
| 51 |
+
document_version_id=document_version_id,
|
| 52 |
+
chunk_index=chunk_index,
|
| 53 |
+
text=chunk_text,
|
| 54 |
+
page_number=metadata.get("page"),
|
| 55 |
+
section=(
|
| 56 |
+
metadata.get("section_title")
|
| 57 |
+
or metadata.get("section_type")
|
| 58 |
+
),
|
| 59 |
+
)
|
| 60 |
+
)
|
| 61 |
+
|
| 62 |
+
chunk_index += 1
|
| 63 |
+
|
| 64 |
+
self.db.commit()
|
| 65 |
+
|
| 66 |
+
return {
|
| 67 |
+
"chunks_created": chunk_index,
|
| 68 |
+
}
|
| 69 |
+
|
| 70 |
+
@classmethod
|
| 71 |
+
def _chunk_text(
|
| 72 |
+
cls,
|
| 73 |
+
text: str,
|
| 74 |
+
) -> list[str]:
|
| 75 |
+
|
| 76 |
+
if len(text) <= cls.CHUNK_SIZE:
|
| 77 |
+
return [text]
|
| 78 |
+
|
| 79 |
+
chunks = []
|
| 80 |
+
|
| 81 |
+
start = 0
|
| 82 |
+
text_length = len(text)
|
| 83 |
+
|
| 84 |
+
while start < text_length:
|
| 85 |
+
|
| 86 |
+
end = min(
|
| 87 |
+
start + cls.CHUNK_SIZE,
|
| 88 |
+
text_length,
|
| 89 |
+
)
|
| 90 |
+
|
| 91 |
+
chunk = text[start:end].strip()
|
| 92 |
+
|
| 93 |
+
if chunk:
|
| 94 |
+
chunks.append(chunk)
|
| 95 |
+
|
| 96 |
+
if end >= text_length:
|
| 97 |
+
break
|
| 98 |
+
|
| 99 |
+
start = end - cls.OVERLAP
|
| 100 |
+
|
| 101 |
+
return chunks
|
backend/app/services/document_processor.py
CHANGED
|
@@ -107,6 +107,65 @@ def clean_text(text: str) -> str:
|
|
| 107 |
text = re.sub(r"\n{3,}", "\n\n", text)
|
| 108 |
return text.strip()
|
| 109 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 110 |
|
| 111 |
def detect_type(file_path: str, mime_type: str = None):
|
| 112 |
extension = os.path.splitext(file_path)[1].lower()
|
|
@@ -285,7 +344,10 @@ def extract_pdf_docling(file_path: str) -> list[TextSection]:
|
|
| 285 |
|
| 286 |
def extract_pdf_ocr(file_path: str) -> list[TextSection]:
|
| 287 |
try:
|
| 288 |
-
images = convert_from_path(
|
|
|
|
|
|
|
|
|
|
| 289 |
sections = []
|
| 290 |
|
| 291 |
for page_number, image in enumerate(images, start=1):
|
|
@@ -305,21 +367,73 @@ def extract_pdf_ocr(file_path: str) -> list[TextSection]:
|
|
| 305 |
|
| 306 |
|
| 307 |
def extract_pdf_sections(file_path: str) -> list[TextSection]:
|
| 308 |
-
"""
|
| 309 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 310 |
|
| 311 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 312 |
return sections
|
| 313 |
|
| 314 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 315 |
sections = extract_pdf_docling(file_path)
|
| 316 |
|
| 317 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 318 |
return sections
|
| 319 |
|
| 320 |
-
|
| 321 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 322 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 323 |
|
| 324 |
# ---------------------------------------------------------------------------
|
| 325 |
# Other format extractors
|
|
|
|
| 107 |
text = re.sub(r"\n{3,}", "\n\n", text)
|
| 108 |
return text.strip()
|
| 109 |
|
| 110 |
+
def extraction_quality(sections: list[TextSection]) -> float:
|
| 111 |
+
"""
|
| 112 |
+
Estimate whether extracted PDF text is substantial enough
|
| 113 |
+
to be considered usable.
|
| 114 |
+
|
| 115 |
+
Returns a score from 0.0 to 1.0.
|
| 116 |
+
"""
|
| 117 |
+
|
| 118 |
+
if not sections:
|
| 119 |
+
return 0.0
|
| 120 |
+
|
| 121 |
+
total_chars = sum(len(section.text) for section in sections)
|
| 122 |
+
|
| 123 |
+
if total_chars == 0:
|
| 124 |
+
return 0.0
|
| 125 |
+
|
| 126 |
+
text = "\n".join(section.text for section in sections)
|
| 127 |
+
|
| 128 |
+
words = re.findall(r"\b\w+\b", text)
|
| 129 |
+
if not words:
|
| 130 |
+
return 0.0
|
| 131 |
+
|
| 132 |
+
alphanumeric_chars = sum(
|
| 133 |
+
char.isalnum()
|
| 134 |
+
for char in text
|
| 135 |
+
)
|
| 136 |
+
|
| 137 |
+
alphanumeric_ratio = alphanumeric_chars / max(len(text), 1)
|
| 138 |
+
|
| 139 |
+
word_count = len(words)
|
| 140 |
+
|
| 141 |
+
score = 0.0
|
| 142 |
+
|
| 143 |
+
# Amount of actual text
|
| 144 |
+
if total_chars >= 2000:
|
| 145 |
+
score += 0.4
|
| 146 |
+
elif total_chars >= 1000:
|
| 147 |
+
score += 0.25
|
| 148 |
+
elif total_chars >= 500:
|
| 149 |
+
score += 0.1
|
| 150 |
+
|
| 151 |
+
# Number of words
|
| 152 |
+
if word_count >= 300:
|
| 153 |
+
score += 0.3
|
| 154 |
+
elif word_count >= 150:
|
| 155 |
+
score += 0.2
|
| 156 |
+
elif word_count >= 75:
|
| 157 |
+
score += 0.1
|
| 158 |
+
|
| 159 |
+
# Mostly actual text rather than symbols/noise
|
| 160 |
+
if alphanumeric_ratio >= 0.75:
|
| 161 |
+
score += 0.3
|
| 162 |
+
elif alphanumeric_ratio >= 0.60:
|
| 163 |
+
score += 0.2
|
| 164 |
+
elif alphanumeric_ratio >= 0.45:
|
| 165 |
+
score += 0.1
|
| 166 |
+
|
| 167 |
+
return min(score, 1.0)
|
| 168 |
+
|
| 169 |
|
| 170 |
def detect_type(file_path: str, mime_type: str = None):
|
| 171 |
extension = os.path.splitext(file_path)[1].lower()
|
|
|
|
| 344 |
|
| 345 |
def extract_pdf_ocr(file_path: str) -> list[TextSection]:
|
| 346 |
try:
|
| 347 |
+
images = convert_from_path(
|
| 348 |
+
file_path,
|
| 349 |
+
poppler_path=os.getenv("POPPLER_PATH"),
|
| 350 |
+
)
|
| 351 |
sections = []
|
| 352 |
|
| 353 |
for page_number, image in enumerate(images, start=1):
|
|
|
|
| 367 |
|
| 368 |
|
| 369 |
def extract_pdf_sections(file_path: str) -> list[TextSection]:
|
| 370 |
+
"""
|
| 371 |
+
Extract PDF text using a quality-aware cascade:
|
| 372 |
+
|
| 373 |
+
PyMuPDF → Docling → OCR
|
| 374 |
+
"""
|
| 375 |
+
|
| 376 |
+
# ------------------------------------------------------------
|
| 377 |
+
# 1. PyMuPDF
|
| 378 |
+
# ------------------------------------------------------------
|
| 379 |
|
| 380 |
+
sections = (
|
| 381 |
+
extract_pdf_text_pymupdf(file_path)
|
| 382 |
+
if fitz
|
| 383 |
+
else extract_pdf_text_pypdf(file_path)
|
| 384 |
+
)
|
| 385 |
+
|
| 386 |
+
score = extraction_quality(sections)
|
| 387 |
+
|
| 388 |
+
logger.info(
|
| 389 |
+
"PyMuPDF extraction quality: %.2f (%s chars)",
|
| 390 |
+
score,
|
| 391 |
+
sum(len(s.text) for s in sections),
|
| 392 |
+
)
|
| 393 |
+
|
| 394 |
+
if score >= 0.6:
|
| 395 |
return sections
|
| 396 |
|
| 397 |
+
# ------------------------------------------------------------
|
| 398 |
+
# 2. Docling
|
| 399 |
+
# ------------------------------------------------------------
|
| 400 |
+
|
| 401 |
+
logger.info(
|
| 402 |
+
"PyMuPDF extraction quality insufficient, trying Docling"
|
| 403 |
+
)
|
| 404 |
+
|
| 405 |
sections = extract_pdf_docling(file_path)
|
| 406 |
|
| 407 |
+
score = extraction_quality(sections)
|
| 408 |
+
|
| 409 |
+
logger.info(
|
| 410 |
+
"Docling extraction quality: %.2f (%s chars)",
|
| 411 |
+
score,
|
| 412 |
+
sum(len(s.text) for s in sections),
|
| 413 |
+
)
|
| 414 |
+
|
| 415 |
+
if score >= 0.6:
|
| 416 |
return sections
|
| 417 |
|
| 418 |
+
# ------------------------------------------------------------
|
| 419 |
+
# 3. OCR
|
| 420 |
+
# ------------------------------------------------------------
|
| 421 |
+
|
| 422 |
+
logger.info(
|
| 423 |
+
"Docling extraction quality insufficient, triggering OCR"
|
| 424 |
+
)
|
| 425 |
|
| 426 |
+
sections = extract_pdf_ocr(file_path)
|
| 427 |
+
|
| 428 |
+
score = extraction_quality(sections)
|
| 429 |
+
|
| 430 |
+
logger.info(
|
| 431 |
+
"OCR extraction quality: %.2f (%s chars)",
|
| 432 |
+
score,
|
| 433 |
+
sum(len(s.text) for s in sections),
|
| 434 |
+
)
|
| 435 |
+
|
| 436 |
+
return sections
|
| 437 |
|
| 438 |
# ---------------------------------------------------------------------------
|
| 439 |
# Other format extractors
|
backend/app/services/knowledge_link_service.py
ADDED
|
@@ -0,0 +1,172 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
from difflib import SequenceMatcher
|
| 4 |
+
|
| 5 |
+
from sqlalchemy.orm import Session
|
| 6 |
+
|
| 7 |
+
from app.models.knowledge_item import KnowledgeItem
|
| 8 |
+
from app.models.knowledge_link import (
|
| 9 |
+
KnowledgeLink,
|
| 10 |
+
RelationshipType,
|
| 11 |
+
)
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
class KnowledgeLinkService:
|
| 15 |
+
"""
|
| 16 |
+
Builds semantic relationships between KnowledgeItems.
|
| 17 |
+
|
| 18 |
+
Link creation is intentionally conservative:
|
| 19 |
+
only strong, deterministic relationships are created.
|
| 20 |
+
"""
|
| 21 |
+
|
| 22 |
+
def __init__(self, db: Session):
|
| 23 |
+
self.db = db
|
| 24 |
+
|
| 25 |
+
def link_document(
|
| 26 |
+
self,
|
| 27 |
+
*,
|
| 28 |
+
workspace_id,
|
| 29 |
+
document_version_id,
|
| 30 |
+
) -> dict:
|
| 31 |
+
current_items = (
|
| 32 |
+
self.db.query(KnowledgeItem)
|
| 33 |
+
.filter(
|
| 34 |
+
KnowledgeItem.workspace_id == workspace_id,
|
| 35 |
+
KnowledgeItem.document_version_id == document_version_id,
|
| 36 |
+
)
|
| 37 |
+
.all()
|
| 38 |
+
)
|
| 39 |
+
|
| 40 |
+
existing_items = (
|
| 41 |
+
self.db.query(KnowledgeItem)
|
| 42 |
+
.filter(
|
| 43 |
+
KnowledgeItem.workspace_id == workspace_id,
|
| 44 |
+
KnowledgeItem.document_version_id != document_version_id,
|
| 45 |
+
)
|
| 46 |
+
.all()
|
| 47 |
+
)
|
| 48 |
+
|
| 49 |
+
created = 0
|
| 50 |
+
|
| 51 |
+
for source in current_items:
|
| 52 |
+
for target in existing_items:
|
| 53 |
+
if source.id == target.id:
|
| 54 |
+
continue
|
| 55 |
+
|
| 56 |
+
relationship = self._infer_relationship(
|
| 57 |
+
source,
|
| 58 |
+
target,
|
| 59 |
+
)
|
| 60 |
+
|
| 61 |
+
if relationship is None:
|
| 62 |
+
continue
|
| 63 |
+
|
| 64 |
+
relationship_type, confidence = relationship
|
| 65 |
+
|
| 66 |
+
if self._link_exists(
|
| 67 |
+
source.id,
|
| 68 |
+
target.id,
|
| 69 |
+
relationship_type,
|
| 70 |
+
):
|
| 71 |
+
continue
|
| 72 |
+
|
| 73 |
+
self.db.add(
|
| 74 |
+
KnowledgeLink(
|
| 75 |
+
source_item_id=source.id,
|
| 76 |
+
target_item_id=target.id,
|
| 77 |
+
relationship_type=relationship_type,
|
| 78 |
+
confidence=confidence,
|
| 79 |
+
)
|
| 80 |
+
)
|
| 81 |
+
|
| 82 |
+
created += 1
|
| 83 |
+
|
| 84 |
+
self.db.commit()
|
| 85 |
+
|
| 86 |
+
return {
|
| 87 |
+
"links_created": created,
|
| 88 |
+
}
|
| 89 |
+
|
| 90 |
+
def _infer_relationship(
|
| 91 |
+
self,
|
| 92 |
+
source: KnowledgeItem,
|
| 93 |
+
target: KnowledgeItem,
|
| 94 |
+
):
|
| 95 |
+
title_similarity = self._similarity(
|
| 96 |
+
source.title,
|
| 97 |
+
target.title,
|
| 98 |
+
)
|
| 99 |
+
|
| 100 |
+
value_similarity = self._similarity(
|
| 101 |
+
source.value,
|
| 102 |
+
target.value,
|
| 103 |
+
)
|
| 104 |
+
|
| 105 |
+
# Same concept with highly similar values.
|
| 106 |
+
if (
|
| 107 |
+
source.type == target.type
|
| 108 |
+
and title_similarity >= 0.85
|
| 109 |
+
and value_similarity >= 0.80
|
| 110 |
+
):
|
| 111 |
+
return (
|
| 112 |
+
RelationshipType.RELATED_TO,
|
| 113 |
+
round(
|
| 114 |
+
(title_similarity + value_similarity) / 2,
|
| 115 |
+
3,
|
| 116 |
+
),
|
| 117 |
+
)
|
| 118 |
+
|
| 119 |
+
# A method/treatment can be related to an entity/claim.
|
| 120 |
+
if (
|
| 121 |
+
source.type.name == "METHOD"
|
| 122 |
+
and target.type.name in {"ENTITY", "CLAIM", "OBSERVATION"}
|
| 123 |
+
and title_similarity >= 0.65
|
| 124 |
+
):
|
| 125 |
+
return (
|
| 126 |
+
RelationshipType.USES,
|
| 127 |
+
round(title_similarity, 3),
|
| 128 |
+
)
|
| 129 |
+
|
| 130 |
+
# Metrics commonly describe claims/observations.
|
| 131 |
+
if (
|
| 132 |
+
source.type.name == "METRIC"
|
| 133 |
+
and target.type.name in {"CLAIM", "OBSERVATION"}
|
| 134 |
+
and title_similarity >= 0.65
|
| 135 |
+
):
|
| 136 |
+
return (
|
| 137 |
+
RelationshipType.REFERENCES,
|
| 138 |
+
round(title_similarity, 3),
|
| 139 |
+
)
|
| 140 |
+
|
| 141 |
+
return None
|
| 142 |
+
|
| 143 |
+
def _link_exists(
|
| 144 |
+
self,
|
| 145 |
+
source_item_id,
|
| 146 |
+
target_item_id,
|
| 147 |
+
relationship_type: RelationshipType,
|
| 148 |
+
) -> bool:
|
| 149 |
+
return (
|
| 150 |
+
self.db.query(KnowledgeLink)
|
| 151 |
+
.filter(
|
| 152 |
+
KnowledgeLink.source_item_id == source_item_id,
|
| 153 |
+
KnowledgeLink.target_item_id == target_item_id,
|
| 154 |
+
KnowledgeLink.relationship_type == relationship_type,
|
| 155 |
+
)
|
| 156 |
+
.first()
|
| 157 |
+
is not None
|
| 158 |
+
)
|
| 159 |
+
|
| 160 |
+
@staticmethod
|
| 161 |
+
def _similarity(
|
| 162 |
+
left: str | None,
|
| 163 |
+
right: str | None,
|
| 164 |
+
) -> float:
|
| 165 |
+
if not left or not right:
|
| 166 |
+
return 0.0
|
| 167 |
+
|
| 168 |
+
return SequenceMatcher(
|
| 169 |
+
None,
|
| 170 |
+
left.lower().strip(),
|
| 171 |
+
right.lower().strip(),
|
| 172 |
+
).ratio()
|
backend/app/services/reconciliation_service.py
ADDED
|
@@ -0,0 +1,223 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
from sqlalchemy.orm import Session
|
| 4 |
+
|
| 5 |
+
from app.models.knowledge_item import KnowledgeItem
|
| 6 |
+
from app.models.proposal import Proposal, ProposalStatus, ProposalType
|
| 7 |
+
from app.repositories.knowledge_repository import KnowledgeRepository
|
| 8 |
+
from difflib import SequenceMatcher
|
| 9 |
+
|
| 10 |
+
class ReconciliationService:
|
| 11 |
+
"""
|
| 12 |
+
Compares newly extracted knowledge against the existing
|
| 13 |
+
Knowledge Register and creates proposals for changes.
|
| 14 |
+
|
| 15 |
+
Reconciliation never directly mutates existing knowledge.
|
| 16 |
+
All changes go through PENDING proposals.
|
| 17 |
+
"""
|
| 18 |
+
|
| 19 |
+
def __init__(self, db: Session):
|
| 20 |
+
self.db = db
|
| 21 |
+
self.repository = KnowledgeRepository(db)
|
| 22 |
+
|
| 23 |
+
def reconcile_document(
|
| 24 |
+
self,
|
| 25 |
+
*,
|
| 26 |
+
workspace_id,
|
| 27 |
+
document_version_id,
|
| 28 |
+
) -> dict:
|
| 29 |
+
new_items = self.repository.list_by_document_version(
|
| 30 |
+
document_version_id
|
| 31 |
+
)
|
| 32 |
+
|
| 33 |
+
existing_items = (
|
| 34 |
+
self.db.query(KnowledgeItem)
|
| 35 |
+
.filter(
|
| 36 |
+
KnowledgeItem.workspace_id == workspace_id,
|
| 37 |
+
KnowledgeItem.document_version_id != document_version_id,
|
| 38 |
+
)
|
| 39 |
+
.all()
|
| 40 |
+
)
|
| 41 |
+
|
| 42 |
+
new_count = 0
|
| 43 |
+
duplicate_count = 0
|
| 44 |
+
conflict_count = 0
|
| 45 |
+
|
| 46 |
+
for item in new_items:
|
| 47 |
+
match = self._find_match(item, existing_items)
|
| 48 |
+
|
| 49 |
+
if match is None:
|
| 50 |
+
self._create_create_proposal(
|
| 51 |
+
workspace_id=workspace_id,
|
| 52 |
+
item=item,
|
| 53 |
+
)
|
| 54 |
+
new_count += 1
|
| 55 |
+
continue
|
| 56 |
+
|
| 57 |
+
if self._is_duplicate(item, match):
|
| 58 |
+
duplicate_count += 1
|
| 59 |
+
continue
|
| 60 |
+
|
| 61 |
+
self._create_update_proposal(
|
| 62 |
+
workspace_id=workspace_id,
|
| 63 |
+
existing_item=match,
|
| 64 |
+
new_item=item,
|
| 65 |
+
)
|
| 66 |
+
conflict_count += 1
|
| 67 |
+
|
| 68 |
+
self.db.commit()
|
| 69 |
+
|
| 70 |
+
return {
|
| 71 |
+
"new": new_count,
|
| 72 |
+
"duplicates": duplicate_count,
|
| 73 |
+
"conflicts": conflict_count,
|
| 74 |
+
}
|
| 75 |
+
|
| 76 |
+
def _find_match(
|
| 77 |
+
self,
|
| 78 |
+
item: KnowledgeItem,
|
| 79 |
+
existing_items: list[KnowledgeItem],
|
| 80 |
+
) -> KnowledgeItem | None:
|
| 81 |
+
|
| 82 |
+
best_match = None
|
| 83 |
+
best_score = 0.0
|
| 84 |
+
|
| 85 |
+
new_title = self._normalize(item.title)
|
| 86 |
+
|
| 87 |
+
for existing in existing_items:
|
| 88 |
+
if existing.type != item.type:
|
| 89 |
+
continue
|
| 90 |
+
|
| 91 |
+
existing_title = self._normalize(existing.title)
|
| 92 |
+
|
| 93 |
+
# Exact match
|
| 94 |
+
if new_title == existing_title:
|
| 95 |
+
return existing
|
| 96 |
+
|
| 97 |
+
# Similarity match
|
| 98 |
+
score = SequenceMatcher(
|
| 99 |
+
None,
|
| 100 |
+
new_title,
|
| 101 |
+
existing_title,
|
| 102 |
+
).ratio()
|
| 103 |
+
|
| 104 |
+
if score > best_score:
|
| 105 |
+
best_score = score
|
| 106 |
+
best_match = existing
|
| 107 |
+
|
| 108 |
+
if best_score >= 0.80:
|
| 109 |
+
return best_match
|
| 110 |
+
|
| 111 |
+
return None
|
| 112 |
+
|
| 113 |
+
def _is_duplicate(
|
| 114 |
+
self,
|
| 115 |
+
new_item: KnowledgeItem,
|
| 116 |
+
existing_item: KnowledgeItem,
|
| 117 |
+
) -> bool:
|
| 118 |
+
|
| 119 |
+
return (
|
| 120 |
+
self._normalize(new_item.value)
|
| 121 |
+
== self._normalize(existing_item.value)
|
| 122 |
+
)
|
| 123 |
+
|
| 124 |
+
def _pending_proposal_exists(
|
| 125 |
+
self,
|
| 126 |
+
knowledge_item_id,
|
| 127 |
+
proposal_type: ProposalType,
|
| 128 |
+
) -> bool:
|
| 129 |
+
|
| 130 |
+
return (
|
| 131 |
+
self.db.query(Proposal)
|
| 132 |
+
.filter(
|
| 133 |
+
Proposal.knowledge_item_id == knowledge_item_id,
|
| 134 |
+
Proposal.proposal_type == proposal_type,
|
| 135 |
+
Proposal.status == ProposalStatus.PENDING,
|
| 136 |
+
)
|
| 137 |
+
.first()
|
| 138 |
+
is not None
|
| 139 |
+
)
|
| 140 |
+
|
| 141 |
+
def _create_create_proposal(
|
| 142 |
+
self,
|
| 143 |
+
*,
|
| 144 |
+
workspace_id,
|
| 145 |
+
item: KnowledgeItem,
|
| 146 |
+
):
|
| 147 |
+
if self._pending_proposal_exists(
|
| 148 |
+
item.id,
|
| 149 |
+
ProposalType.CREATE,
|
| 150 |
+
):
|
| 151 |
+
return
|
| 152 |
+
|
| 153 |
+
proposal = Proposal(
|
| 154 |
+
workspace_id=workspace_id,
|
| 155 |
+
knowledge_item_id=item.id,
|
| 156 |
+
proposal_type=ProposalType.CREATE,
|
| 157 |
+
status=ProposalStatus.PENDING,
|
| 158 |
+
summary=f"Create knowledge item: {item.title}",
|
| 159 |
+
rationale=(
|
| 160 |
+
"No matching knowledge item was found in the "
|
| 161 |
+
"existing Knowledge Register."
|
| 162 |
+
),
|
| 163 |
+
proposed_changes={
|
| 164 |
+
"type": item.type.value,
|
| 165 |
+
"title": item.title,
|
| 166 |
+
"value": item.value,
|
| 167 |
+
"summary": item.summary,
|
| 168 |
+
"attributes": item.attributes,
|
| 169 |
+
"confidence": item.confidence,
|
| 170 |
+
},
|
| 171 |
+
)
|
| 172 |
+
|
| 173 |
+
self.db.add(proposal)
|
| 174 |
+
|
| 175 |
+
def _create_update_proposal(
|
| 176 |
+
self,
|
| 177 |
+
*,
|
| 178 |
+
workspace_id,
|
| 179 |
+
existing_item: KnowledgeItem,
|
| 180 |
+
new_item: KnowledgeItem,
|
| 181 |
+
):
|
| 182 |
+
if self._pending_proposal_exists(
|
| 183 |
+
existing_item.id,
|
| 184 |
+
ProposalType.UPDATE,
|
| 185 |
+
):
|
| 186 |
+
return
|
| 187 |
+
|
| 188 |
+
proposal = Proposal(
|
| 189 |
+
workspace_id=workspace_id,
|
| 190 |
+
knowledge_item_id=existing_item.id,
|
| 191 |
+
proposal_type=ProposalType.UPDATE,
|
| 192 |
+
status=ProposalStatus.PENDING,
|
| 193 |
+
summary=f"Update knowledge item: {existing_item.title}",
|
| 194 |
+
rationale=(
|
| 195 |
+
"A matching knowledge item exists, but the newly "
|
| 196 |
+
"extracted value differs from the registered value."
|
| 197 |
+
),
|
| 198 |
+
proposed_changes={
|
| 199 |
+
"existing": {
|
| 200 |
+
"value": existing_item.value,
|
| 201 |
+
"summary": existing_item.summary,
|
| 202 |
+
"attributes": existing_item.attributes,
|
| 203 |
+
"confidence": existing_item.confidence,
|
| 204 |
+
},
|
| 205 |
+
"proposed": {
|
| 206 |
+
"value": new_item.value,
|
| 207 |
+
"summary": new_item.summary,
|
| 208 |
+
"attributes": new_item.attributes,
|
| 209 |
+
"confidence": new_item.confidence,
|
| 210 |
+
},
|
| 211 |
+
"source_knowledge_item_id": str(new_item.id),
|
| 212 |
+
},
|
| 213 |
+
)
|
| 214 |
+
|
| 215 |
+
self.db.add(proposal)
|
| 216 |
+
@staticmethod
|
| 217 |
+
def _normalize(value: str | None) -> str:
|
| 218 |
+
if not value:
|
| 219 |
+
return ""
|
| 220 |
+
|
| 221 |
+
words = value.lower().strip().split()
|
| 222 |
+
|
| 223 |
+
return " ".join(sorted(words))
|
backend/app/workflow/executor.py
CHANGED
|
@@ -2,25 +2,19 @@ from __future__ import annotations
|
|
| 2 |
|
| 3 |
from sqlalchemy.orm import Session
|
| 4 |
|
| 5 |
-
from app.llm.client import LLMClient
|
| 6 |
-
from app.repositories.knowledge_repository import KnowledgeRepository
|
| 7 |
from app.services.workflow_service import WorkflowService
|
| 8 |
-
from app.workflow.
|
| 9 |
-
from app.workflow.nodes.complete import complete
|
| 10 |
-
from app.workflow.nodes.extract import extract
|
| 11 |
-
from app.workflow.nodes.knowledge import knowledge
|
| 12 |
from app.workflow.state import WorkflowState
|
| 13 |
|
| 14 |
|
| 15 |
class WorkflowExecutor:
|
| 16 |
"""
|
| 17 |
-
Executes the document workflow.
|
| 18 |
"""
|
| 19 |
|
| 20 |
def __init__(self, db: Session):
|
| 21 |
self.workflow_service = WorkflowService(db)
|
| 22 |
-
self.
|
| 23 |
-
self.knowledge_repository = KnowledgeRepository(db)
|
| 24 |
|
| 25 |
def execute(
|
| 26 |
self,
|
|
@@ -29,21 +23,12 @@ class WorkflowExecutor:
|
|
| 29 |
) -> WorkflowState:
|
| 30 |
|
| 31 |
try:
|
| 32 |
-
|
| 33 |
-
|
| 34 |
-
state = classify(state)
|
| 35 |
-
|
| 36 |
-
state = knowledge(
|
| 37 |
-
state,
|
| 38 |
-
self.llm_client,
|
| 39 |
-
self.knowledge_repository,
|
| 40 |
-
)
|
| 41 |
-
|
| 42 |
-
state = complete(state)
|
| 43 |
|
| 44 |
self.workflow_service.complete_workflow(workflow)
|
| 45 |
|
| 46 |
-
return
|
| 47 |
|
| 48 |
except Exception:
|
|
|
|
| 49 |
raise
|
|
|
|
| 2 |
|
| 3 |
from sqlalchemy.orm import Session
|
| 4 |
|
|
|
|
|
|
|
| 5 |
from app.services.workflow_service import WorkflowService
|
| 6 |
+
from app.workflow.graph import build_workflow
|
|
|
|
|
|
|
|
|
|
| 7 |
from app.workflow.state import WorkflowState
|
| 8 |
|
| 9 |
|
| 10 |
class WorkflowExecutor:
|
| 11 |
"""
|
| 12 |
+
Executes the document workflow through LangGraph.
|
| 13 |
"""
|
| 14 |
|
| 15 |
def __init__(self, db: Session):
|
| 16 |
self.workflow_service = WorkflowService(db)
|
| 17 |
+
self.graph = build_workflow(db)
|
|
|
|
| 18 |
|
| 19 |
def execute(
|
| 20 |
self,
|
|
|
|
| 23 |
) -> WorkflowState:
|
| 24 |
|
| 25 |
try:
|
| 26 |
+
final_state = self.graph.invoke(state)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 27 |
|
| 28 |
self.workflow_service.complete_workflow(workflow)
|
| 29 |
|
| 30 |
+
return final_state
|
| 31 |
|
| 32 |
except Exception:
|
| 33 |
+
self.workflow_service.fail_workflow(workflow)
|
| 34 |
raise
|
backend/app/workflow/graph.py
CHANGED
|
@@ -0,0 +1,89 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
from sqlalchemy.orm import Session
|
| 4 |
+
from langgraph.graph import END, START, StateGraph
|
| 5 |
+
|
| 6 |
+
from app.llm.client import LLMClient
|
| 7 |
+
from app.repositories.knowledge_repository import KnowledgeRepository
|
| 8 |
+
from app.services.chunking_service import ChunkingService
|
| 9 |
+
from app.services.knowledge_link_service import KnowledgeLinkService
|
| 10 |
+
|
| 11 |
+
from app.workflow.nodes.chunk import chunk
|
| 12 |
+
from app.workflow.nodes.classify import classify
|
| 13 |
+
from app.workflow.nodes.complete import complete
|
| 14 |
+
from app.workflow.nodes.extract import extract
|
| 15 |
+
from app.workflow.nodes.knowledge import knowledge
|
| 16 |
+
from app.workflow.nodes.link import link
|
| 17 |
+
from app.workflow.nodes.reconcile import reconcile
|
| 18 |
+
from app.workflow.state import WorkflowState
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
def build_workflow(db: Session):
|
| 22 |
+
"""
|
| 23 |
+
Build the DocWeave document intelligence workflow.
|
| 24 |
+
"""
|
| 25 |
+
|
| 26 |
+
llm_client = LLMClient()
|
| 27 |
+
knowledge_repository = KnowledgeRepository(db)
|
| 28 |
+
chunking_service = ChunkingService(db)
|
| 29 |
+
knowledge_link_service = KnowledgeLinkService(db)
|
| 30 |
+
|
| 31 |
+
graph = StateGraph(WorkflowState)
|
| 32 |
+
|
| 33 |
+
# ------------------------------------------------------------------
|
| 34 |
+
# Nodes
|
| 35 |
+
# ------------------------------------------------------------------
|
| 36 |
+
|
| 37 |
+
graph.add_node("extract", extract)
|
| 38 |
+
|
| 39 |
+
graph.add_node(
|
| 40 |
+
"chunk",
|
| 41 |
+
lambda state: chunk(
|
| 42 |
+
state,
|
| 43 |
+
chunking_service,
|
| 44 |
+
),
|
| 45 |
+
)
|
| 46 |
+
|
| 47 |
+
graph.add_node("classify", classify)
|
| 48 |
+
|
| 49 |
+
graph.add_node(
|
| 50 |
+
"knowledge",
|
| 51 |
+
lambda state: knowledge(
|
| 52 |
+
state,
|
| 53 |
+
llm_client,
|
| 54 |
+
knowledge_repository,
|
| 55 |
+
),
|
| 56 |
+
)
|
| 57 |
+
|
| 58 |
+
graph.add_node(
|
| 59 |
+
"reconciliation",
|
| 60 |
+
lambda state: reconcile(
|
| 61 |
+
state,
|
| 62 |
+
db,
|
| 63 |
+
),
|
| 64 |
+
)
|
| 65 |
+
|
| 66 |
+
graph.add_node(
|
| 67 |
+
"link",
|
| 68 |
+
lambda state: link(
|
| 69 |
+
state,
|
| 70 |
+
knowledge_link_service,
|
| 71 |
+
),
|
| 72 |
+
)
|
| 73 |
+
|
| 74 |
+
graph.add_node("complete", complete)
|
| 75 |
+
|
| 76 |
+
# ------------------------------------------------------------------
|
| 77 |
+
# Workflow edges
|
| 78 |
+
# ------------------------------------------------------------------
|
| 79 |
+
|
| 80 |
+
graph.add_edge(START, "extract")
|
| 81 |
+
graph.add_edge("extract", "chunk")
|
| 82 |
+
graph.add_edge("chunk", "classify")
|
| 83 |
+
graph.add_edge("classify", "knowledge")
|
| 84 |
+
graph.add_edge("knowledge", "reconciliation")
|
| 85 |
+
graph.add_edge("reconciliation", "link")
|
| 86 |
+
graph.add_edge("link", "complete")
|
| 87 |
+
graph.add_edge("complete", END)
|
| 88 |
+
|
| 89 |
+
return graph.compile()
|
backend/app/workflow/nodes/chunk.py
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from app.services.chunking_service import ChunkingService
|
| 2 |
+
from app.workflow.state import WorkflowState
|
| 3 |
+
|
| 4 |
+
|
| 5 |
+
def chunk(
|
| 6 |
+
state: WorkflowState,
|
| 7 |
+
chunking_service: ChunkingService,
|
| 8 |
+
) -> WorkflowState:
|
| 9 |
+
"""
|
| 10 |
+
Persist retrieval-ready document chunks while preserving
|
| 11 |
+
lightweight provenance.
|
| 12 |
+
"""
|
| 13 |
+
|
| 14 |
+
state.current_node = "CHUNKING"
|
| 15 |
+
|
| 16 |
+
result = chunking_service.chunk_document(
|
| 17 |
+
document_version_id=state.document_version_id,
|
| 18 |
+
sections=state.extracted_sections,
|
| 19 |
+
)
|
| 20 |
+
|
| 21 |
+
state.metadata["chunks_created"] = result["chunks_created"]
|
| 22 |
+
|
| 23 |
+
return state
|
backend/app/workflow/nodes/link.py
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from app.services.knowledge_link_service import KnowledgeLinkService
|
| 2 |
+
from app.workflow.state import WorkflowState
|
| 3 |
+
|
| 4 |
+
|
| 5 |
+
def link(
|
| 6 |
+
state: WorkflowState,
|
| 7 |
+
knowledge_link_service: KnowledgeLinkService,
|
| 8 |
+
) -> WorkflowState:
|
| 9 |
+
"""
|
| 10 |
+
Create semantic links between the newly extracted knowledge
|
| 11 |
+
and the existing Knowledge Register.
|
| 12 |
+
"""
|
| 13 |
+
|
| 14 |
+
state.current_node = "KNOWLEDGE_LINKING"
|
| 15 |
+
|
| 16 |
+
result = knowledge_link_service.link_document(
|
| 17 |
+
workspace_id=state.workspace_id,
|
| 18 |
+
document_version_id=state.document_version_id,
|
| 19 |
+
)
|
| 20 |
+
|
| 21 |
+
state.metadata["knowledge_links_created"] = (
|
| 22 |
+
result["links_created"]
|
| 23 |
+
)
|
| 24 |
+
|
| 25 |
+
return state
|
backend/app/workflow/nodes/reconcile.py
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
from sqlalchemy.orm import Session
|
| 4 |
+
|
| 5 |
+
from app.services.reconciliation_service import ReconciliationService
|
| 6 |
+
from app.workflow.state import WorkflowState
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
def reconcile(
|
| 10 |
+
state: WorkflowState,
|
| 11 |
+
db: Session,
|
| 12 |
+
) -> WorkflowState:
|
| 13 |
+
"""
|
| 14 |
+
Reconcile newly extracted knowledge against the
|
| 15 |
+
existing Knowledge Register.
|
| 16 |
+
"""
|
| 17 |
+
|
| 18 |
+
state.current_node = "RECONCILIATION"
|
| 19 |
+
|
| 20 |
+
service = ReconciliationService(db)
|
| 21 |
+
|
| 22 |
+
results = service.reconcile_document(
|
| 23 |
+
workspace_id=state.workspace_id,
|
| 24 |
+
document_version_id=state.document_version_id,
|
| 25 |
+
)
|
| 26 |
+
|
| 27 |
+
state.metadata["reconciliation"] = results
|
| 28 |
+
|
| 29 |
+
return state
|