tarzanagh commited on
Commit
92844c3
·
verified ·
1 Parent(s): 90f1cca

Fix: re-extract empty OpenIE cache

Browse files
Files changed (1) hide show
  1. src/passage_entity/kg_builder.py +9 -1
src/passage_entity/kg_builder.py CHANGED
@@ -388,7 +388,15 @@ class KGBuilder:
388
  # Standardise indices
389
  for item in all_info:
390
  item["idx"] = compute_mdhash_id(item["passage"], "chunk-")
391
- existing_keys = {info["idx"] for info in all_info}
 
 
 
 
 
 
 
 
392
  for ck in chunk_keys:
393
  if ck not in existing_keys:
394
  chunk_keys_to_save.add(ck)
 
388
  # Standardise indices
389
  for item in all_info:
390
  item["idx"] = compute_mdhash_id(item["passage"], "chunk-")
391
+ # Only consider a doc "done" if it has at least some extracted content
392
+ existing_keys = set()
393
+ for info in all_info:
394
+ has_content = (
395
+ len(info.get("extracted_entities", [])) > 0
396
+ or len(info.get("extracted_triples", [])) > 0
397
+ )
398
+ if has_content:
399
+ existing_keys.add(info["idx"])
400
  for ck in chunk_keys:
401
  if ck not in existing_keys:
402
  chunk_keys_to_save.add(ck)