yugbirla commited on
Commit
08cea84
·
1 Parent(s): b75b281

Add source viewer for citation evidence

Browse files
app/deployment/hf_status.py CHANGED
@@ -4058,7 +4058,7 @@ function updateCitations(data) {
4058
  <div class="source-line"><b>Chunk:</b> ${source.chunk_id}</div>
4059
  <div class="preview-text">${String(source.preview).slice(0, 260)}</div>
4060
  <br>
4061
- <button class="light" onclick="openSourceModal(${index})">View source details</button>
4062
  `;
4063
 
4064
  box.appendChild(card);
 
4058
  <div class="source-line"><b>Chunk:</b> ${source.chunk_id}</div>
4059
  <div class="preview-text">${String(source.preview).slice(0, 260)}</div>
4060
  <br>
4061
+ <button class="light" onclick="openSourceModal(${index})">Open source details</button>
4062
  `;
4063
 
4064
  box.appendChild(card);
app/main.py CHANGED
@@ -1,3 +1,4 @@
 
1
  from app.deployment.hf_status import get_home_html, get_product_app_html
2
  from app.deployment.hf_status import get_product_app_html
3
  import uuid
@@ -656,3 +657,35 @@ def product_app_page():
656
  @app.get("/app", response_class=HTMLResponse)
657
  def product_workspace_app_page():
658
  return get_product_app_html()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from app.product.source_viewer import get_source_details, get_source_html
2
  from app.deployment.hf_status import get_home_html, get_product_app_html
3
  from app.deployment.hf_status import get_product_app_html
4
  import uuid
 
657
  @app.get("/app", response_class=HTMLResponse)
658
  def product_workspace_app_page():
659
  return get_product_app_html()
660
+
661
+
662
+ # Source viewer endpoints
663
+
664
+ @app.get("/documents/{document_id}/sources/{source_id}")
665
+ def document_source_details(
666
+ document_id: str,
667
+ source_id: str,
668
+ page: str = "",
669
+ chunk_id: str = ""
670
+ ):
671
+ return get_source_details(
672
+ document_id=document_id,
673
+ source_id=source_id,
674
+ page=page,
675
+ chunk_id=chunk_id
676
+ )
677
+
678
+
679
+ @app.get("/documents/{document_id}/sources/{source_id}/view", response_class=HTMLResponse)
680
+ def document_source_view(
681
+ document_id: str,
682
+ source_id: str,
683
+ page: str = "",
684
+ chunk_id: str = ""
685
+ ):
686
+ return get_source_html(
687
+ document_id=document_id,
688
+ source_id=source_id,
689
+ page=page,
690
+ chunk_id=chunk_id
691
+ )
app/product/source_viewer.py ADDED
@@ -0,0 +1,430 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+
2
+ import json
3
+ import csv
4
+ from pathlib import Path
5
+ from typing import Dict, Any, List, Optional
6
+
7
+ from fastapi import HTTPException
8
+ from fastapi.responses import HTMLResponse
9
+
10
+ from app.core.config import settings
11
+
12
+
13
+ def safe_str(value) -> str:
14
+ if value is None:
15
+ return ""
16
+ return str(value)
17
+
18
+
19
+ def html_escape(value: str) -> str:
20
+ return (
21
+ safe_str(value)
22
+ .replace("&", "&amp;")
23
+ .replace("<", "&lt;")
24
+ .replace(">", "&gt;")
25
+ .replace('"', "&quot;")
26
+ )
27
+
28
+
29
+ def normalize(value) -> str:
30
+ return safe_str(value).strip().lower()
31
+
32
+
33
+ def get_processed_document_dir(document_id: str) -> Path:
34
+ return Path(settings.PROCESSED_DIR) / document_id
35
+
36
+
37
+ def load_json_file(path: Path):
38
+ try:
39
+ return json.loads(path.read_text(encoding="utf-8"))
40
+ except Exception:
41
+ try:
42
+ return json.loads(path.read_text(encoding="utf-8-sig"))
43
+ except Exception:
44
+ return None
45
+
46
+
47
+ def load_jsonl_file(path: Path) -> List[Dict[str, Any]]:
48
+ rows = []
49
+
50
+ try:
51
+ lines = path.read_text(encoding="utf-8").splitlines()
52
+ except Exception:
53
+ try:
54
+ lines = path.read_text(encoding="utf-8-sig").splitlines()
55
+ except Exception:
56
+ return rows
57
+
58
+ for line in lines:
59
+ line = line.strip()
60
+ if not line:
61
+ continue
62
+
63
+ try:
64
+ item = json.loads(line)
65
+ if isinstance(item, dict):
66
+ rows.append(item)
67
+ except Exception:
68
+ pass
69
+
70
+ return rows
71
+
72
+
73
+ def load_csv_file(path: Path) -> List[Dict[str, Any]]:
74
+ rows = []
75
+
76
+ for enc in ["utf-8", "utf-8-sig"]:
77
+ try:
78
+ with path.open("r", encoding=enc, newline="") as f:
79
+ reader = csv.DictReader(f)
80
+ for row in reader:
81
+ rows.append(dict(row))
82
+ return rows
83
+ except Exception:
84
+ rows = []
85
+
86
+ return rows
87
+
88
+
89
+ def flatten_json_records(data) -> List[Dict[str, Any]]:
90
+ records = []
91
+
92
+ if isinstance(data, dict):
93
+ for key in ["chunks", "results", "pages", "items", "documents", "data"]:
94
+ if isinstance(data.get(key), list):
95
+ for item in data[key]:
96
+ if isinstance(item, dict):
97
+ records.append(item)
98
+
99
+ if not records:
100
+ records.append(data)
101
+
102
+ elif isinstance(data, list):
103
+ for item in data:
104
+ if isinstance(item, dict):
105
+ records.append(item)
106
+
107
+ return records
108
+
109
+
110
+ def collect_candidate_records(document_id: str) -> List[Dict[str, Any]]:
111
+ doc_dir = get_processed_document_dir(document_id)
112
+ processed_dir = Path(settings.PROCESSED_DIR)
113
+
114
+ roots = []
115
+
116
+ if doc_dir.exists():
117
+ roots.append(doc_dir)
118
+
119
+ if processed_dir.exists():
120
+ roots.append(processed_dir)
121
+
122
+ records = []
123
+ seen_files = set()
124
+
125
+ for root in roots:
126
+ for path in root.rglob("*"):
127
+ if not path.is_file():
128
+ continue
129
+
130
+ if path in seen_files:
131
+ continue
132
+
133
+ seen_files.add(path)
134
+
135
+ suffix = path.suffix.lower()
136
+ file_records = []
137
+
138
+ if suffix == ".json":
139
+ file_records = flatten_json_records(load_json_file(path))
140
+ elif suffix == ".jsonl":
141
+ file_records = load_jsonl_file(path)
142
+ elif suffix == ".csv":
143
+ file_records = load_csv_file(path)
144
+
145
+ for record in file_records:
146
+ enriched = dict(record)
147
+ enriched["_source_file_path"] = str(path)
148
+ records.append(enriched)
149
+
150
+ return records
151
+
152
+
153
+ def value_from(record: Dict[str, Any], keys: List[str], default: str = "") -> str:
154
+ for key in keys:
155
+ if key in record and record[key] not in [None, ""]:
156
+ return safe_str(record[key])
157
+
158
+ metadata = record.get("metadata")
159
+
160
+ if isinstance(metadata, dict):
161
+ for key in keys:
162
+ if key in metadata and metadata[key] not in [None, ""]:
163
+ return safe_str(metadata[key])
164
+
165
+ return default
166
+
167
+
168
+ def record_text(record: Dict[str, Any]) -> str:
169
+ return value_from(
170
+ record,
171
+ [
172
+ "text",
173
+ "content",
174
+ "chunk_text",
175
+ "page_text",
176
+ "cleaned_text",
177
+ "raw_text",
178
+ "body",
179
+ "preview",
180
+ "text_preview",
181
+ "chunk_preview"
182
+ ],
183
+ ""
184
+ )
185
+
186
+
187
+ def record_match_score(
188
+ record: Dict[str, Any],
189
+ source_id: str,
190
+ page: Optional[str] = None,
191
+ chunk_id: Optional[str] = None
192
+ ) -> int:
193
+ score = 0
194
+
195
+ source_id_norm = normalize(source_id)
196
+ page_norm = normalize(page)
197
+ chunk_id_norm = normalize(chunk_id)
198
+
199
+ candidate_source_values = [
200
+ value_from(record, ["source_id", "citation_id", "id", "source"]),
201
+ value_from(record, ["chunk_id", "chunk", "chunk_index", "chunk_number"]),
202
+ value_from(record, ["page_id", "page_source_id"])
203
+ ]
204
+
205
+ candidate_page_values = [
206
+ value_from(record, ["page", "page_number", "page_no", "page_index"])
207
+ ]
208
+
209
+ candidate_chunk_values = [
210
+ value_from(record, ["chunk_id", "chunk", "chunk_index", "chunk_number", "id"])
211
+ ]
212
+
213
+ if source_id_norm:
214
+ for value in candidate_source_values:
215
+ value_norm = normalize(value)
216
+
217
+ if value_norm == source_id_norm:
218
+ score += 10
219
+ elif source_id_norm in value_norm or value_norm in source_id_norm:
220
+ score += 3
221
+
222
+ if page_norm:
223
+ for value in candidate_page_values:
224
+ if normalize(value) == page_norm:
225
+ score += 5
226
+
227
+ if chunk_id_norm:
228
+ for value in candidate_chunk_values:
229
+ if normalize(value) == chunk_id_norm:
230
+ score += 8
231
+
232
+ if record_text(record):
233
+ score += 1
234
+
235
+ return score
236
+
237
+
238
+ def find_best_source_record(
239
+ document_id: str,
240
+ source_id: str,
241
+ page: Optional[str] = None,
242
+ chunk_id: Optional[str] = None
243
+ ) -> Dict[str, Any]:
244
+ records = collect_candidate_records(document_id)
245
+
246
+ if not records:
247
+ raise HTTPException(
248
+ status_code=404,
249
+ detail="No processed records found. Upload/index the document first."
250
+ )
251
+
252
+ scored = []
253
+
254
+ for record in records:
255
+ score = record_match_score(
256
+ record=record,
257
+ source_id=source_id,
258
+ page=page,
259
+ chunk_id=chunk_id
260
+ )
261
+ scored.append((score, record))
262
+
263
+ scored.sort(key=lambda item: item[0], reverse=True)
264
+
265
+ best_score, best_record = scored[0]
266
+
267
+ if best_score <= 0:
268
+ raise HTTPException(
269
+ status_code=404,
270
+ detail="Source record not found."
271
+ )
272
+
273
+ return best_record
274
+
275
+
276
+ def get_source_details(
277
+ document_id: str,
278
+ source_id: str,
279
+ page: Optional[str] = None,
280
+ chunk_id: Optional[str] = None
281
+ ) -> Dict[str, Any]:
282
+ record = find_best_source_record(
283
+ document_id=document_id,
284
+ source_id=source_id,
285
+ page=page,
286
+ chunk_id=chunk_id
287
+ )
288
+
289
+ document_name = value_from(
290
+ record,
291
+ ["document_name", "source_file_name", "file_name", "filename", "document_title"],
292
+ "Selected document"
293
+ )
294
+
295
+ page_number = value_from(
296
+ record,
297
+ ["page", "page_number", "page_no", "page_index"],
298
+ page or "Not available"
299
+ )
300
+
301
+ resolved_chunk_id = value_from(
302
+ record,
303
+ ["chunk_id", "chunk", "chunk_index", "chunk_number", "id"],
304
+ chunk_id or source_id
305
+ )
306
+
307
+ text = record_text(record)
308
+
309
+ return {
310
+ "document_id": document_id,
311
+ "source_id": source_id,
312
+ "document_name": document_name,
313
+ "page": page_number,
314
+ "chunk_id": resolved_chunk_id,
315
+ "text": text,
316
+ "text_preview": text[:1200],
317
+ "metadata": record,
318
+ "source_file_path": record.get("_source_file_path")
319
+ }
320
+
321
+
322
+ def get_source_html(
323
+ document_id: str,
324
+ source_id: str,
325
+ page: Optional[str] = None,
326
+ chunk_id: Optional[str] = None
327
+ ) -> HTMLResponse:
328
+ details = get_source_details(
329
+ document_id=document_id,
330
+ source_id=source_id,
331
+ page=page,
332
+ chunk_id=chunk_id
333
+ )
334
+
335
+ document_name = html_escape(details.get("document_name", "Selected document"))
336
+ page_value = html_escape(details.get("page", "Not available"))
337
+ chunk_value = html_escape(details.get("chunk_id", source_id))
338
+ text_value = html_escape(details.get("text", "Source text not available."))
339
+ metadata_value = html_escape(json.dumps(details.get("metadata", {}), indent=2, ensure_ascii=False))
340
+
341
+ html = f'''
342
+ <!DOCTYPE html>
343
+ <html>
344
+ <head>
345
+ <title>Source {html_escape(source_id)} - GraphResearcher</title>
346
+ <meta name="viewport" content="width=device-width, initial-scale=1">
347
+
348
+ <style>
349
+ body {{
350
+ font-family: Inter, Arial, sans-serif;
351
+ background: #f8fafc;
352
+ color: #0f172a;
353
+ margin: 0;
354
+ padding: 32px;
355
+ }}
356
+
357
+ .container {{
358
+ max-width: 980px;
359
+ margin: 0 auto;
360
+ }}
361
+
362
+ .card {{
363
+ background: white;
364
+ border: 1px solid #e5e7eb;
365
+ border-radius: 18px;
366
+ padding: 22px;
367
+ margin-bottom: 18px;
368
+ box-shadow: 0 1px 4px rgba(0,0,0,0.04);
369
+ }}
370
+
371
+ .pill {{
372
+ display: inline-block;
373
+ background: #eef2ff;
374
+ color: #3730a3;
375
+ padding: 6px 10px;
376
+ border-radius: 999px;
377
+ font-size: 13px;
378
+ margin: 4px 5px 4px 0;
379
+ }}
380
+
381
+ pre {{
382
+ white-space: pre-wrap;
383
+ word-break: break-word;
384
+ background: #0f172a;
385
+ color: #e5e7eb;
386
+ padding: 16px;
387
+ border-radius: 14px;
388
+ line-height: 1.55;
389
+ }}
390
+
391
+ .source-text {{
392
+ white-space: pre-wrap;
393
+ line-height: 1.75;
394
+ font-size: 16px;
395
+ }}
396
+
397
+ a {{
398
+ color: #2563eb;
399
+ font-weight: 800;
400
+ text-decoration: none;
401
+ }}
402
+ </style>
403
+ </head>
404
+
405
+ <body>
406
+ <div class="container">
407
+ <p><a href="/app">← Back to app</a></p>
408
+
409
+ <div class="card">
410
+ <h1>Source {html_escape(source_id)}</h1>
411
+ <span class="pill">Document: {document_name}</span>
412
+ <span class="pill">Page: {page_value}</span>
413
+ <span class="pill">Chunk: {chunk_value}</span>
414
+ </div>
415
+
416
+ <div class="card">
417
+ <h2>Evidence Text</h2>
418
+ <div class="source-text">{text_value or "Source text not available."}</div>
419
+ </div>
420
+
421
+ <div class="card">
422
+ <h2>Raw Metadata</h2>
423
+ <pre>{metadata_value}</pre>
424
+ </div>
425
+ </div>
426
+ </body>
427
+ </html>
428
+ '''
429
+
430
+ return HTMLResponse(content=html)
scripts/phase28_source_viewer_clickable_citations.py ADDED
@@ -0,0 +1,513 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from pathlib import Path
2
+ import re
3
+
4
+ # Clean BOM
5
+ for path in Path("app").rglob("*.py"):
6
+ text = path.read_text(encoding="utf-8-sig")
7
+ text = text.replace("\ufeff", "")
8
+ path.write_text(text, encoding="utf-8")
9
+
10
+ Path("app/product").mkdir(parents=True, exist_ok=True)
11
+
12
+ # -----------------------------------------------------
13
+ # 1. Create source_viewer.py
14
+ # -----------------------------------------------------
15
+
16
+ Path("app/product/source_viewer.py").write_text(r"""
17
+ import json
18
+ import csv
19
+ from pathlib import Path
20
+ from typing import Dict, Any, List, Optional
21
+
22
+ from fastapi import HTTPException
23
+ from fastapi.responses import HTMLResponse
24
+
25
+ from app.core.config import settings
26
+
27
+
28
+ def safe_str(value) -> str:
29
+ if value is None:
30
+ return ""
31
+ return str(value)
32
+
33
+
34
+ def html_escape(value: str) -> str:
35
+ return (
36
+ safe_str(value)
37
+ .replace("&", "&amp;")
38
+ .replace("<", "&lt;")
39
+ .replace(">", "&gt;")
40
+ .replace('"', "&quot;")
41
+ )
42
+
43
+
44
+ def normalize(value) -> str:
45
+ return safe_str(value).strip().lower()
46
+
47
+
48
+ def get_processed_document_dir(document_id: str) -> Path:
49
+ return Path(settings.PROCESSED_DIR) / document_id
50
+
51
+
52
+ def load_json_file(path: Path):
53
+ try:
54
+ return json.loads(path.read_text(encoding="utf-8"))
55
+ except Exception:
56
+ try:
57
+ return json.loads(path.read_text(encoding="utf-8-sig"))
58
+ except Exception:
59
+ return None
60
+
61
+
62
+ def load_jsonl_file(path: Path) -> List[Dict[str, Any]]:
63
+ rows = []
64
+
65
+ try:
66
+ lines = path.read_text(encoding="utf-8").splitlines()
67
+ except Exception:
68
+ try:
69
+ lines = path.read_text(encoding="utf-8-sig").splitlines()
70
+ except Exception:
71
+ return rows
72
+
73
+ for line in lines:
74
+ line = line.strip()
75
+ if not line:
76
+ continue
77
+
78
+ try:
79
+ item = json.loads(line)
80
+ if isinstance(item, dict):
81
+ rows.append(item)
82
+ except Exception:
83
+ pass
84
+
85
+ return rows
86
+
87
+
88
+ def load_csv_file(path: Path) -> List[Dict[str, Any]]:
89
+ rows = []
90
+
91
+ for enc in ["utf-8", "utf-8-sig"]:
92
+ try:
93
+ with path.open("r", encoding=enc, newline="") as f:
94
+ reader = csv.DictReader(f)
95
+ for row in reader:
96
+ rows.append(dict(row))
97
+ return rows
98
+ except Exception:
99
+ rows = []
100
+
101
+ return rows
102
+
103
+
104
+ def flatten_json_records(data) -> List[Dict[str, Any]]:
105
+ records = []
106
+
107
+ if isinstance(data, dict):
108
+ for key in ["chunks", "results", "pages", "items", "documents", "data"]:
109
+ if isinstance(data.get(key), list):
110
+ for item in data[key]:
111
+ if isinstance(item, dict):
112
+ records.append(item)
113
+
114
+ if not records:
115
+ records.append(data)
116
+
117
+ elif isinstance(data, list):
118
+ for item in data:
119
+ if isinstance(item, dict):
120
+ records.append(item)
121
+
122
+ return records
123
+
124
+
125
+ def collect_candidate_records(document_id: str) -> List[Dict[str, Any]]:
126
+ doc_dir = get_processed_document_dir(document_id)
127
+ processed_dir = Path(settings.PROCESSED_DIR)
128
+
129
+ roots = []
130
+
131
+ if doc_dir.exists():
132
+ roots.append(doc_dir)
133
+
134
+ if processed_dir.exists():
135
+ roots.append(processed_dir)
136
+
137
+ records = []
138
+ seen_files = set()
139
+
140
+ for root in roots:
141
+ for path in root.rglob("*"):
142
+ if not path.is_file():
143
+ continue
144
+
145
+ if path in seen_files:
146
+ continue
147
+
148
+ seen_files.add(path)
149
+
150
+ suffix = path.suffix.lower()
151
+ file_records = []
152
+
153
+ if suffix == ".json":
154
+ file_records = flatten_json_records(load_json_file(path))
155
+ elif suffix == ".jsonl":
156
+ file_records = load_jsonl_file(path)
157
+ elif suffix == ".csv":
158
+ file_records = load_csv_file(path)
159
+
160
+ for record in file_records:
161
+ enriched = dict(record)
162
+ enriched["_source_file_path"] = str(path)
163
+ records.append(enriched)
164
+
165
+ return records
166
+
167
+
168
+ def value_from(record: Dict[str, Any], keys: List[str], default: str = "") -> str:
169
+ for key in keys:
170
+ if key in record and record[key] not in [None, ""]:
171
+ return safe_str(record[key])
172
+
173
+ metadata = record.get("metadata")
174
+
175
+ if isinstance(metadata, dict):
176
+ for key in keys:
177
+ if key in metadata and metadata[key] not in [None, ""]:
178
+ return safe_str(metadata[key])
179
+
180
+ return default
181
+
182
+
183
+ def record_text(record: Dict[str, Any]) -> str:
184
+ return value_from(
185
+ record,
186
+ [
187
+ "text",
188
+ "content",
189
+ "chunk_text",
190
+ "page_text",
191
+ "cleaned_text",
192
+ "raw_text",
193
+ "body",
194
+ "preview",
195
+ "text_preview",
196
+ "chunk_preview"
197
+ ],
198
+ ""
199
+ )
200
+
201
+
202
+ def record_match_score(
203
+ record: Dict[str, Any],
204
+ source_id: str,
205
+ page: Optional[str] = None,
206
+ chunk_id: Optional[str] = None
207
+ ) -> int:
208
+ score = 0
209
+
210
+ source_id_norm = normalize(source_id)
211
+ page_norm = normalize(page)
212
+ chunk_id_norm = normalize(chunk_id)
213
+
214
+ candidate_source_values = [
215
+ value_from(record, ["source_id", "citation_id", "id", "source"]),
216
+ value_from(record, ["chunk_id", "chunk", "chunk_index", "chunk_number"]),
217
+ value_from(record, ["page_id", "page_source_id"])
218
+ ]
219
+
220
+ candidate_page_values = [
221
+ value_from(record, ["page", "page_number", "page_no", "page_index"])
222
+ ]
223
+
224
+ candidate_chunk_values = [
225
+ value_from(record, ["chunk_id", "chunk", "chunk_index", "chunk_number", "id"])
226
+ ]
227
+
228
+ if source_id_norm:
229
+ for value in candidate_source_values:
230
+ value_norm = normalize(value)
231
+
232
+ if value_norm == source_id_norm:
233
+ score += 10
234
+ elif source_id_norm in value_norm or value_norm in source_id_norm:
235
+ score += 3
236
+
237
+ if page_norm:
238
+ for value in candidate_page_values:
239
+ if normalize(value) == page_norm:
240
+ score += 5
241
+
242
+ if chunk_id_norm:
243
+ for value in candidate_chunk_values:
244
+ if normalize(value) == chunk_id_norm:
245
+ score += 8
246
+
247
+ if record_text(record):
248
+ score += 1
249
+
250
+ return score
251
+
252
+
253
+ def find_best_source_record(
254
+ document_id: str,
255
+ source_id: str,
256
+ page: Optional[str] = None,
257
+ chunk_id: Optional[str] = None
258
+ ) -> Dict[str, Any]:
259
+ records = collect_candidate_records(document_id)
260
+
261
+ if not records:
262
+ raise HTTPException(
263
+ status_code=404,
264
+ detail="No processed records found. Upload/index the document first."
265
+ )
266
+
267
+ scored = []
268
+
269
+ for record in records:
270
+ score = record_match_score(
271
+ record=record,
272
+ source_id=source_id,
273
+ page=page,
274
+ chunk_id=chunk_id
275
+ )
276
+ scored.append((score, record))
277
+
278
+ scored.sort(key=lambda item: item[0], reverse=True)
279
+
280
+ best_score, best_record = scored[0]
281
+
282
+ if best_score <= 0:
283
+ raise HTTPException(
284
+ status_code=404,
285
+ detail="Source record not found."
286
+ )
287
+
288
+ return best_record
289
+
290
+
291
+ def get_source_details(
292
+ document_id: str,
293
+ source_id: str,
294
+ page: Optional[str] = None,
295
+ chunk_id: Optional[str] = None
296
+ ) -> Dict[str, Any]:
297
+ record = find_best_source_record(
298
+ document_id=document_id,
299
+ source_id=source_id,
300
+ page=page,
301
+ chunk_id=chunk_id
302
+ )
303
+
304
+ document_name = value_from(
305
+ record,
306
+ ["document_name", "source_file_name", "file_name", "filename", "document_title"],
307
+ "Selected document"
308
+ )
309
+
310
+ page_number = value_from(
311
+ record,
312
+ ["page", "page_number", "page_no", "page_index"],
313
+ page or "Not available"
314
+ )
315
+
316
+ resolved_chunk_id = value_from(
317
+ record,
318
+ ["chunk_id", "chunk", "chunk_index", "chunk_number", "id"],
319
+ chunk_id or source_id
320
+ )
321
+
322
+ text = record_text(record)
323
+
324
+ return {
325
+ "document_id": document_id,
326
+ "source_id": source_id,
327
+ "document_name": document_name,
328
+ "page": page_number,
329
+ "chunk_id": resolved_chunk_id,
330
+ "text": text,
331
+ "text_preview": text[:1200],
332
+ "metadata": record,
333
+ "source_file_path": record.get("_source_file_path")
334
+ }
335
+
336
+
337
+ def get_source_html(
338
+ document_id: str,
339
+ source_id: str,
340
+ page: Optional[str] = None,
341
+ chunk_id: Optional[str] = None
342
+ ) -> HTMLResponse:
343
+ details = get_source_details(
344
+ document_id=document_id,
345
+ source_id=source_id,
346
+ page=page,
347
+ chunk_id=chunk_id
348
+ )
349
+
350
+ document_name = html_escape(details.get("document_name", "Selected document"))
351
+ page_value = html_escape(details.get("page", "Not available"))
352
+ chunk_value = html_escape(details.get("chunk_id", source_id))
353
+ text_value = html_escape(details.get("text", "Source text not available."))
354
+ metadata_value = html_escape(json.dumps(details.get("metadata", {}), indent=2, ensure_ascii=False))
355
+
356
+ html = f'''
357
+ <!DOCTYPE html>
358
+ <html>
359
+ <head>
360
+ <title>Source {html_escape(source_id)} - GraphResearcher</title>
361
+ <meta name="viewport" content="width=device-width, initial-scale=1">
362
+
363
+ <style>
364
+ body {{
365
+ font-family: Inter, Arial, sans-serif;
366
+ background: #f8fafc;
367
+ color: #0f172a;
368
+ margin: 0;
369
+ padding: 32px;
370
+ }}
371
+
372
+ .container {{
373
+ max-width: 980px;
374
+ margin: 0 auto;
375
+ }}
376
+
377
+ .card {{
378
+ background: white;
379
+ border: 1px solid #e5e7eb;
380
+ border-radius: 18px;
381
+ padding: 22px;
382
+ margin-bottom: 18px;
383
+ box-shadow: 0 1px 4px rgba(0,0,0,0.04);
384
+ }}
385
+
386
+ .pill {{
387
+ display: inline-block;
388
+ background: #eef2ff;
389
+ color: #3730a3;
390
+ padding: 6px 10px;
391
+ border-radius: 999px;
392
+ font-size: 13px;
393
+ margin: 4px 5px 4px 0;
394
+ }}
395
+
396
+ pre {{
397
+ white-space: pre-wrap;
398
+ word-break: break-word;
399
+ background: #0f172a;
400
+ color: #e5e7eb;
401
+ padding: 16px;
402
+ border-radius: 14px;
403
+ line-height: 1.55;
404
+ }}
405
+
406
+ .source-text {{
407
+ white-space: pre-wrap;
408
+ line-height: 1.75;
409
+ font-size: 16px;
410
+ }}
411
+
412
+ a {{
413
+ color: #2563eb;
414
+ font-weight: 800;
415
+ text-decoration: none;
416
+ }}
417
+ </style>
418
+ </head>
419
+
420
+ <body>
421
+ <div class="container">
422
+ <p><a href="/app">← Back to app</a></p>
423
+
424
+ <div class="card">
425
+ <h1>Source {html_escape(source_id)}</h1>
426
+ <span class="pill">Document: {document_name}</span>
427
+ <span class="pill">Page: {page_value}</span>
428
+ <span class="pill">Chunk: {chunk_value}</span>
429
+ </div>
430
+
431
+ <div class="card">
432
+ <h2>Evidence Text</h2>
433
+ <div class="source-text">{text_value or "Source text not available."}</div>
434
+ </div>
435
+
436
+ <div class="card">
437
+ <h2>Raw Metadata</h2>
438
+ <pre>{metadata_value}</pre>
439
+ </div>
440
+ </div>
441
+ </body>
442
+ </html>
443
+ '''
444
+
445
+ return HTMLResponse(content=html)
446
+ """, encoding="utf-8")
447
+
448
+
449
+ # -----------------------------------------------------
450
+ # 2. Patch main.py
451
+ # -----------------------------------------------------
452
+
453
+ main_path = Path("app/main.py")
454
+ main_text = main_path.read_text(encoding="utf-8-sig")
455
+ main_text = main_text.replace("\ufeff", "")
456
+
457
+ if "from app.product.source_viewer import" not in main_text:
458
+ main_text = (
459
+ "from app.product.source_viewer import get_source_details, get_source_html\n"
460
+ + main_text
461
+ )
462
+
463
+ if "# Source viewer endpoints" not in main_text:
464
+ main_text += '''
465
+
466
+ # Source viewer endpoints
467
+
468
+ @app.get("/documents/{document_id}/sources/{source_id}")
469
+ def document_source_details(
470
+ document_id: str,
471
+ source_id: str,
472
+ page: str = "",
473
+ chunk_id: str = ""
474
+ ):
475
+ return get_source_details(
476
+ document_id=document_id,
477
+ source_id=source_id,
478
+ page=page,
479
+ chunk_id=chunk_id
480
+ )
481
+
482
+
483
+ @app.get("/documents/{document_id}/sources/{source_id}/view", response_class=HTMLResponse)
484
+ def document_source_view(
485
+ document_id: str,
486
+ source_id: str,
487
+ page: str = "",
488
+ chunk_id: str = ""
489
+ ):
490
+ return get_source_html(
491
+ document_id=document_id,
492
+ source_id=source_id,
493
+ page=page,
494
+ chunk_id=chunk_id
495
+ )
496
+ '''
497
+
498
+ main_path.write_text(main_text, encoding="utf-8")
499
+
500
+
501
+ # -----------------------------------------------------
502
+ # 3. Patch app UI button text only
503
+ # -----------------------------------------------------
504
+
505
+ hf_path = Path("app/deployment/hf_status.py")
506
+ ui_text = hf_path.read_text(encoding="utf-8-sig")
507
+ ui_text = ui_text.replace("\ufeff", "")
508
+
509
+ ui_text = ui_text.replace("View source details", "Open source details")
510
+
511
+ hf_path.write_text(ui_text, encoding="utf-8")
512
+
513
+ print("Fixed Phase 28 source viewer patch complete.")