Fix: re-extract empty OpenIE cache
Browse files
src/passage_entity/kg_builder.py
CHANGED
|
@@ -388,7 +388,15 @@ class KGBuilder:
|
|
| 388 |
# Standardise indices
|
| 389 |
for item in all_info:
|
| 390 |
item["idx"] = compute_mdhash_id(item["passage"], "chunk-")
|
| 391 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 392 |
for ck in chunk_keys:
|
| 393 |
if ck not in existing_keys:
|
| 394 |
chunk_keys_to_save.add(ck)
|
|
|
|
| 388 |
# Standardise indices
|
| 389 |
for item in all_info:
|
| 390 |
item["idx"] = compute_mdhash_id(item["passage"], "chunk-")
|
| 391 |
+
# Only consider a doc "done" if it has at least some extracted content
|
| 392 |
+
existing_keys = set()
|
| 393 |
+
for info in all_info:
|
| 394 |
+
has_content = (
|
| 395 |
+
len(info.get("extracted_entities", [])) > 0
|
| 396 |
+
or len(info.get("extracted_triples", [])) > 0
|
| 397 |
+
)
|
| 398 |
+
if has_content:
|
| 399 |
+
existing_keys.add(info["idx"])
|
| 400 |
for ck in chunk_keys:
|
| 401 |
if ck not in existing_keys:
|
| 402 |
chunk_keys_to_save.add(ck)
|