shak3008 commited on
Commit
b53b3e8
·
1 Parent(s): 652930e

feat: end-to-end cost tracking with frontend metrics panel

Browse files

Backend:
- Token tracking wired into knowledge extraction node
(reads langchain response_metadata for prompt/completion tokens)
- Stage timing added to extract, chunk, embed, reconcile nodes
- Each stage start/end recorded via RunTracker

Frontend:
- New api/metrics.js module
- Document Workspace shows 'Run cost and timing' card after workflow
completes or pauses for review
- Summary row: total time, tokens used, LLM calls
- Per-stage breakdown with proportional bar chart + time + token count
- Stages shown: extraction, chunking, embedding, knowledge_extraction,
reconciliation

The reviewer can now see exactly where time was spent and how many
tokens each run consumed — stage by stage.

backend/app/workflow/nodes/chunk.py CHANGED
@@ -1,4 +1,5 @@
1
  from app.services.chunking_service import ChunkingService
 
2
  from app.workflow.state import WorkflowState
3
 
4
 
@@ -13,6 +14,9 @@ def chunk(
13
 
14
  state.current_node = "CHUNKING"
15
 
 
 
 
16
  result = chunking_service.chunk_document(
17
  document_version_id=state.document_version_id,
18
  sections=state.extracted_sections,
@@ -20,4 +24,5 @@ def chunk(
20
 
21
  state.metadata["chunks_created"] = result["chunks_created"]
22
 
 
23
  return state
 
1
  from app.services.chunking_service import ChunkingService
2
+ from app.core.tracing.run_tracker import get_or_create_tracker
3
  from app.workflow.state import WorkflowState
4
 
5
 
 
14
 
15
  state.current_node = "CHUNKING"
16
 
17
+ tracker = get_or_create_tracker(str(state.workflow_run_id))
18
+ tracker.start_stage("chunking")
19
+
20
  result = chunking_service.chunk_document(
21
  document_version_id=state.document_version_id,
22
  sections=state.extracted_sections,
 
24
 
25
  state.metadata["chunks_created"] = result["chunks_created"]
26
 
27
+ tracker.end_stage("chunking")
28
  return state
backend/app/workflow/nodes/embed.py CHANGED
@@ -2,6 +2,7 @@ from __future__ import annotations
2
 
3
  from sqlalchemy.orm import Session
4
 
 
5
  from app.models.document_chunk import DocumentChunk
6
  from app.services.embedding_service import EmbeddingService
7
  from app.workflow.state import WorkflowState
@@ -18,6 +19,9 @@ def embed(
18
 
19
  state.current_node = "EMBEDDING"
20
 
 
 
 
21
  chunks = (
22
  db.query(DocumentChunk)
23
  .filter(
@@ -31,6 +35,7 @@ def embed(
31
 
32
  if not chunks:
33
  state.metadata["embeddings_created"] = 0
 
34
  return state
35
 
36
  texts = [chunk.text for chunk in chunks]
@@ -38,6 +43,7 @@ def embed(
38
  vectors = embedding_service.embed_many(texts)
39
 
40
  if len(vectors) != len(chunks):
 
41
  raise RuntimeError(
42
  "Embedding count does not match chunk count."
43
  )
@@ -49,4 +55,5 @@ def embed(
49
 
50
  state.metadata["embeddings_created"] = len(chunks)
51
 
 
52
  return state
 
2
 
3
  from sqlalchemy.orm import Session
4
 
5
+ from app.core.tracing.run_tracker import get_or_create_tracker
6
  from app.models.document_chunk import DocumentChunk
7
  from app.services.embedding_service import EmbeddingService
8
  from app.workflow.state import WorkflowState
 
19
 
20
  state.current_node = "EMBEDDING"
21
 
22
+ tracker = get_or_create_tracker(str(state.workflow_run_id))
23
+ tracker.start_stage("embedding")
24
+
25
  chunks = (
26
  db.query(DocumentChunk)
27
  .filter(
 
35
 
36
  if not chunks:
37
  state.metadata["embeddings_created"] = 0
38
+ tracker.end_stage("embedding")
39
  return state
40
 
41
  texts = [chunk.text for chunk in chunks]
 
43
  vectors = embedding_service.embed_many(texts)
44
 
45
  if len(vectors) != len(chunks):
46
+ tracker.end_stage("embedding")
47
  raise RuntimeError(
48
  "Embedding count does not match chunk count."
49
  )
 
55
 
56
  state.metadata["embeddings_created"] = len(chunks)
57
 
58
+ tracker.end_stage("embedding")
59
  return state
backend/app/workflow/nodes/extract.py CHANGED
@@ -1,3 +1,4 @@
 
1
  from app.services.document_processor import (
2
  TextExtractionError,
3
  extract_text_sections,
@@ -13,6 +14,9 @@ def extract(state: WorkflowState) -> WorkflowState:
13
 
14
  state.current_node = "EXTRACTION"
15
 
 
 
 
16
  try:
17
  sections = extract_text_sections(
18
  state.document_path
@@ -34,6 +38,8 @@ def extract(state: WorkflowState) -> WorkflowState:
34
 
35
  except TextExtractionError as exc:
36
  state.metadata["extraction_error"] = str(exc)
 
37
  raise
38
 
 
39
  return state
 
1
+ from app.core.tracing.run_tracker import get_or_create_tracker
2
  from app.services.document_processor import (
3
  TextExtractionError,
4
  extract_text_sections,
 
14
 
15
  state.current_node = "EXTRACTION"
16
 
17
+ tracker = get_or_create_tracker(str(state.workflow_run_id))
18
+ tracker.start_stage("extraction")
19
+
20
  try:
21
  sections = extract_text_sections(
22
  state.document_path
 
38
 
39
  except TextExtractionError as exc:
40
  state.metadata["extraction_error"] = str(exc)
41
+ tracker.end_stage("extraction")
42
  raise
43
 
44
+ tracker.end_stage("extraction")
45
  return state
backend/app/workflow/nodes/knowledge.py CHANGED
@@ -129,9 +129,29 @@ def knowledge(
129
  KnowledgeExtractionResult
130
  )
131
 
132
- result: KnowledgeExtractionResult = structured_model.invoke(
133
- prompt
134
- )
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
135
 
136
  created_count = 0
137
 
 
129
  KnowledgeExtractionResult
130
  )
131
 
132
+ # Track LLM usage
133
+ from app.core.tracing.run_tracker import get_or_create_tracker
134
+ tracker = get_or_create_tracker(str(state.workflow_run_id))
135
+ tracker.start_stage("knowledge_extraction")
136
+
137
+ response = structured_model.invoke(prompt)
138
+
139
+ # Extract token usage from LangChain response metadata
140
+ try:
141
+ if hasattr(response, "response_metadata"):
142
+ meta = response.response_metadata or {}
143
+ usage = meta.get("token_usage") or meta.get("usage") or {}
144
+ tracker.record_llm_usage(
145
+ "knowledge_extraction",
146
+ input_tokens=usage.get("prompt_tokens", 0) or usage.get("input_tokens", 0),
147
+ output_tokens=usage.get("completion_tokens", 0) or usage.get("output_tokens", 0),
148
+ )
149
+ except Exception:
150
+ pass # Non-fatal — don't break workflow for metrics
151
+
152
+ tracker.end_stage("knowledge_extraction")
153
+
154
+ result = response
155
 
156
  created_count = 0
157
 
backend/app/workflow/nodes/reconcile.py CHANGED
@@ -2,6 +2,7 @@ from __future__ import annotations
2
 
3
  from sqlalchemy.orm import Session
4
 
 
5
  from app.services.reconciliation_service import ReconciliationService
6
  from app.workflow.state import WorkflowState
7
 
@@ -17,6 +18,9 @@ def reconcile(
17
 
18
  state.current_node = "RECONCILIATION"
19
 
 
 
 
20
  service = ReconciliationService(db)
21
 
22
  results = service.reconcile_document(
@@ -26,4 +30,5 @@ def reconcile(
26
 
27
  state.metadata["reconciliation"] = results
28
 
 
29
  return state
 
2
 
3
  from sqlalchemy.orm import Session
4
 
5
+ from app.core.tracing.run_tracker import get_or_create_tracker
6
  from app.services.reconciliation_service import ReconciliationService
7
  from app.workflow.state import WorkflowState
8
 
 
18
 
19
  state.current_node = "RECONCILIATION"
20
 
21
+ tracker = get_or_create_tracker(str(state.workflow_run_id))
22
+ tracker.start_stage("reconciliation")
23
+
24
  service = ReconciliationService(db)
25
 
26
  results = service.reconcile_document(
 
30
 
31
  state.metadata["reconciliation"] = results
32
 
33
+ tracker.end_stage("reconciliation")
34
  return state
frontend/src/api/metrics.js ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ import { apiRequest } from "./client";
2
+
3
+ export async function getWorkflowMetrics(workflowId) {
4
+ return apiRequest(`/metrics/workflow/${workflowId}`);
5
+ }
frontend/src/pages/DocumentWorkspace.css CHANGED
@@ -14,3 +14,89 @@
14
  color: var(--color-text-secondary);
15
  margin: 0 0 var(--space-3);
16
  }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
14
  color: var(--color-text-secondary);
15
  margin: 0 0 var(--space-3);
16
  }
17
+
18
+ /* Run cost and timing metrics */
19
+ .dw-metrics__summary {
20
+ display: flex;
21
+ gap: var(--space-6);
22
+ margin-bottom: var(--space-4);
23
+ }
24
+
25
+ .dw-metrics__stat {
26
+ display: flex;
27
+ flex-direction: column;
28
+ gap: var(--space-1);
29
+ }
30
+
31
+ .dw-metrics__stat-value {
32
+ font-size: var(--text-xl);
33
+ font-weight: 700;
34
+ color: var(--color-text-primary);
35
+ }
36
+
37
+ .dw-metrics__stat-label {
38
+ font-size: var(--text-xs);
39
+ color: var(--color-text-muted);
40
+ }
41
+
42
+ .dw-metrics__stages {
43
+ border-top: 1px solid var(--color-border);
44
+ padding-top: var(--space-3);
45
+ }
46
+
47
+ .dw-metrics__stages-title {
48
+ font-size: var(--text-xs);
49
+ font-weight: 600;
50
+ color: var(--color-text-muted);
51
+ text-transform: uppercase;
52
+ letter-spacing: 0.03em;
53
+ margin: 0 0 var(--space-3);
54
+ }
55
+
56
+ .dw-metrics__stage-row {
57
+ display: flex;
58
+ align-items: center;
59
+ gap: var(--space-3);
60
+ padding: var(--space-1) 0;
61
+ }
62
+
63
+ .dw-metrics__stage-name {
64
+ flex-shrink: 0;
65
+ width: 140px;
66
+ font-size: var(--text-xs);
67
+ color: var(--color-text-secondary);
68
+ text-transform: capitalize;
69
+ }
70
+
71
+ .dw-metrics__stage-bar-track {
72
+ flex: 1;
73
+ height: 6px;
74
+ border-radius: 3px;
75
+ background: var(--color-border);
76
+ overflow: hidden;
77
+ }
78
+
79
+ .dw-metrics__stage-bar-fill {
80
+ height: 100%;
81
+ border-radius: 3px;
82
+ background: var(--color-accent);
83
+ min-width: 2px;
84
+ transition: width var(--transition-base);
85
+ }
86
+
87
+ .dw-metrics__stage-time {
88
+ flex-shrink: 0;
89
+ width: 60px;
90
+ text-align: right;
91
+ font-size: var(--text-xs);
92
+ font-family: var(--font-mono);
93
+ color: var(--color-text-muted);
94
+ }
95
+
96
+ .dw-metrics__stage-tokens {
97
+ flex-shrink: 0;
98
+ width: 70px;
99
+ text-align: right;
100
+ font-size: var(--text-xs);
101
+ color: var(--color-text-muted);
102
+ }
frontend/src/pages/DocumentWorkspace.jsx CHANGED
@@ -1,6 +1,7 @@
1
  import { useCallback, useEffect, useRef, useState } from "react";
2
  import { useParams } from "react-router-dom";
3
  import { getWorkflow } from "../api/workflows";
 
4
  import { WorkflowStatusBadge } from "../components/workflow/WorkflowStatusBadge";
5
  import { ProposalReviewList } from "../components/proposals/ProposalReviewList";
6
  import { Card, CardHeader, CardBody, LoadingState, EmptyState, Glossary } from "../components/ui";
@@ -27,6 +28,7 @@ export default function DocumentWorkspace() {
27
  const [workflow, setWorkflow] = useState(null);
28
  const [loading, setLoading] = useState(true);
29
  const [error, setError] = useState(null);
 
30
  const pollRef = useRef(null);
31
 
32
  const fetchWorkflow = useCallback(async () => {
@@ -39,6 +41,11 @@ export default function DocumentWorkspace() {
39
  } else {
40
  setWorkflow(result);
41
  setError(null);
 
 
 
 
 
42
  }
43
  } catch (err) {
44
  setError("Couldn't load workflow status.");
@@ -164,6 +171,73 @@ export default function DocumentWorkspace() {
164
  </CardBody>
165
  </Card>
166
  )}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
167
  </div>
168
  );
169
  }
 
1
  import { useCallback, useEffect, useRef, useState } from "react";
2
  import { useParams } from "react-router-dom";
3
  import { getWorkflow } from "../api/workflows";
4
+ import { getWorkflowMetrics } from "../api/metrics";
5
  import { WorkflowStatusBadge } from "../components/workflow/WorkflowStatusBadge";
6
  import { ProposalReviewList } from "../components/proposals/ProposalReviewList";
7
  import { Card, CardHeader, CardBody, LoadingState, EmptyState, Glossary } from "../components/ui";
 
28
  const [workflow, setWorkflow] = useState(null);
29
  const [loading, setLoading] = useState(true);
30
  const [error, setError] = useState(null);
31
+ const [metrics, setMetrics] = useState(null);
32
  const pollRef = useRef(null);
33
 
34
  const fetchWorkflow = useCallback(async () => {
 
41
  } else {
42
  setWorkflow(result);
43
  setError(null);
44
+ // Fetch metrics when workflow is no longer pending/running
45
+ if (result.status && !ACTIVE_STATUSES.has(result.status)) {
46
+ const m = await getWorkflowMetrics(workflowId);
47
+ if (m && !m.detail) setMetrics(m);
48
+ }
49
  }
50
  } catch (err) {
51
  setError("Couldn't load workflow status.");
 
171
  </CardBody>
172
  </Card>
173
  )}
174
+
175
+ {metrics && metrics.stages && metrics.stages.length > 0 && (
176
+ <Card>
177
+ <CardHeader title="Run cost and timing" />
178
+ <CardBody>
179
+ <div className="dw-metrics">
180
+ <div className="dw-metrics__summary">
181
+ {metrics.total_elapsed_ms != null && (
182
+ <div className="dw-metrics__stat">
183
+ <span className="dw-metrics__stat-value">
184
+ {metrics.total_elapsed_ms > 1000
185
+ ? `${(metrics.total_elapsed_ms / 1000).toFixed(1)}s`
186
+ : `${Math.round(metrics.total_elapsed_ms)}ms`}
187
+ </span>
188
+ <span className="dw-metrics__stat-label">Total time</span>
189
+ </div>
190
+ )}
191
+ {metrics.total_tokens > 0 && (
192
+ <div className="dw-metrics__stat">
193
+ <span className="dw-metrics__stat-value">
194
+ {metrics.total_tokens.toLocaleString()}
195
+ </span>
196
+ <span className="dw-metrics__stat-label">Tokens used</span>
197
+ </div>
198
+ )}
199
+ {metrics.total_llm_calls > 0 && (
200
+ <div className="dw-metrics__stat">
201
+ <span className="dw-metrics__stat-value">
202
+ {metrics.total_llm_calls}
203
+ </span>
204
+ <span className="dw-metrics__stat-label">LLM calls</span>
205
+ </div>
206
+ )}
207
+ </div>
208
+ <div className="dw-metrics__stages">
209
+ <p className="dw-metrics__stages-title">Time per stage</p>
210
+ {metrics.stages.map((stage) => (
211
+ <div key={stage.name} className="dw-metrics__stage-row">
212
+ <span className="dw-metrics__stage-name">{stage.name}</span>
213
+ <div className="dw-metrics__stage-bar-track">
214
+ <div
215
+ className="dw-metrics__stage-bar-fill"
216
+ style={{
217
+ width: `${Math.min(
218
+ 100,
219
+ (stage.elapsed_ms / (metrics.total_elapsed_ms || 1)) * 100
220
+ )}%`,
221
+ }}
222
+ />
223
+ </div>
224
+ <span className="dw-metrics__stage-time">
225
+ {stage.elapsed_ms > 1000
226
+ ? `${(stage.elapsed_ms / 1000).toFixed(1)}s`
227
+ : `${Math.round(stage.elapsed_ms)}ms`}
228
+ </span>
229
+ {stage.input_tokens > 0 && (
230
+ <span className="dw-metrics__stage-tokens">
231
+ {(stage.input_tokens + stage.output_tokens).toLocaleString()} tok
232
+ </span>
233
+ )}
234
+ </div>
235
+ ))}
236
+ </div>
237
+ </div>
238
+ </CardBody>
239
+ </Card>
240
+ )}
241
  </div>
242
  );
243
  }