Create utilities.py

utilities.py

@@ -0,0 +1,20 @@
+import wikipedia
+import re
+
+def clean_and_truncate(text, max_chars=2000):
+    """Clean text and truncate to avoid excessive length."""
+    text = re.sub(r'\n{2,}', '\n', text)  # remove extra newlines
+    return text[:max_chars] + ("..." if len(text) > max_chars else "")
+
+def extract_year_range(text, start_year, end_year):
+    """Extract only content that mentions years in a certain range."""
+    pattern = rf"\b(19[0-9]{{2}}|20[0-2][0-9])\b"
+    matches = re.finditer(pattern, text)
+    filtered_paragraphs = []
+    for match in matches:
+        year = int(match.group())
+        if start_year <= year <= end_year:
+            # capture paragraph
+            para = text[max(0, text.rfind('\n', 0, match.start())) : text.find('\n', match.end())]
+            filtered_paragraphs.append(para.strip())
+    return "\n\n".join(filtered_paragraphs)