Update utils/metadata.py

utils/metadata.py

@@ -18,14 +18,15 @@ def clean_text(text):
 
 def extract_metadata(text):
     """
-    Extract entities: DATE, PERSON, ORGANIZATION, LOCATION
+    Extract named entities from long documents using chunked NER.
     """
     if not text.strip():
         return {"error": "No input provided."}
 
-    text = clean_text(text)[:4000]  # Allow a broader context
-
-    ner_results = ner_pipeline(text)
+    text = clean_text(text)
+    max_chunk_length = 512  # safe for transformer models
+    words = text.split()
+    chunks = [" ".join(words[i:i + max_chunk_length]) for i in range(0, len(words), max_chunk_length)]
 
     metadata = {
         "DATE": [],
@@ -34,11 +35,13 @@ def extract_metadata(text):
         "LOCATION": []
     }
 
-    for ent in ner_results:
-        label = ent["entity_group"]
-        word = ent["word"]
-
-        if label in metadata and word not in metadata[label]:
-            metadata[label].append(word)
+    for chunk in chunks:
+        ner_results = ner_pipeline(chunk)
+        for ent in ner_results:
+            label = ent["entity_group"]
+            word = ent["word"]
+            if label in metadata and word not in metadata[label]:
+                metadata[label].append(word)
 
     return metadata
+